yiciheng/homeutils
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

UI自动化能力优化

Browse Source
This commit is contained in:
张家训
2026-07-08 18:04:48 +08:00
parent 9fdfb05b17
commit 39f2a7e17e
7 changed files with 2361 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/example/workutils/biz/mobileuiauto/MobileUiAutomationPlugin.java
+2
View File
@@ -52,6 +52,7 @@ public class MobileUiAutomationPlugin implements Plugin {
}
latch = new CountDownLatch(1);
window = new MobileUiAutomationWindow(automationService, properties, latch);
window.onFirstShow();
embeddedRoot = window.getContentPane();
window.getStage().getScene().setRoot(new Pane());
return embeddedRoot;
@@ -66,6 +67,7 @@ public class MobileUiAutomationPlugin implements Plugin {
if (embeddedRoot != null && window.getStage().getScene().getRoot() != embeddedRoot) {
window.getStage().getScene().setRoot(embeddedRoot);
}
window.onFirstShow();
window.show();
new Thread(() -> {
try {
src/main/java/com/example/workutils/biz/mobileuiauto/MobileUiAutomationWindow.java
+1225 -72
View File
File diff suppressed because it is too large Load Diff
src/main/java/com/example/workutils/biz/mobileuiauto/service/MobileAutomationService.java
+683 -63
View File
@@ -9,6 +9,7 @@ import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -24,9 +25,12 @@ import java.util.concurrent.TimeUnit;
public class MobileAutomationService {
private static final DateTimeFormatter RUN_ID_FMT = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss");
private static final DateTimeFormatter CASE_NAME_FMT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH-mm");
private static final DateTimeFormatter REPORT_DIR_FMT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH-mm-ss");
private static final String[] PYTHON_RESOURCES = {
"mobile-ui-auto/python/runner.py",
"mobile-ui-auto/python/requirements.txt"
"mobile-ui-auto/python/requirements.txt",
"mobile-ui-auto/python/install_tools.py"
};
private static final String[] TEMPLATE_RESOURCES = {
"mobile-ui-auto/templates/sample-login-flow.yaml"
@@ -41,6 +45,9 @@ public class MobileAutomationService {
private final Map<String, AutomationRunHandle> running = new ConcurrentHashMap<>();
private RecordingSession recordingSession;
// 默认等待时间（毫秒），可通过 UI 配置修改
private int defaultWaitTimeMs = 500;
@PostConstruct
public void init() {
@@ -73,6 +80,67 @@ public class MobileAutomationService {
return venvDir;
}
/**
* 设置默认等待时间（毫秒）
*/
public void setDefaultWaitTime(int waitMs) {
this.defaultWaitTimeMs = Math.max(100, waitMs); // 最小 100ms
}
/**
* 获取当前默认等待时间
*/
public int getDefaultWaitTime() {
return defaultWaitTimeMs;
}
/**
* 列出执行历史记录（从报告目录读取 result.json）
* @param limit 最大返回数量
* @return 按时间倒序排列的历史记录
*/
public List<ExecutionHistoryEntry> listExecutionHistory(int limit) {
List<ExecutionHistoryEntry> entries = new ArrayList<>();
if (!Files.isDirectory(reportsDir)) {
return entries;
}
try (DirectoryStream<Path> stream = Files.newDirectoryStream(reportsDir)) {
for (Path dir : stream) {
if (!Files.isDirectory(dir)) continue;
Path resultFile = dir.resolve("result.json");
if (!Files.exists(resultFile)) continue;
try {
String content = new String(Files.readAllBytes(resultFile), StandardCharsets.UTF_8);
JSONObject json = JSON.parseObject(content);
ExecutionHistoryEntry entry = new ExecutionHistoryEntry();
entry.reportDir = dir;
entry.caseName = json.getString("caseName");
entry.device = json.getString("device");
entry.packageName = json.getString("package");
entry.success = Boolean.TRUE.equals(json.getBoolean("success"));
entry.startTime = json.getString("startTime");
entry.endTime = json.getString("endTime");
entry.steps = json.getJSONArray("steps") != null ? json.getJSONArray("steps").toJavaList(Object.class) : Collections.emptyList();
entries.add(entry);
} catch (Exception e) {
log.debug("解析报告失败: {} - {}", resultFile, e.getMessage());
}
}
} catch (IOException e) {
log.warn("读取报告目录失败: {}", e.getMessage());
}
// 按开始时间倒序排列
entries.sort((a, b) -> {
String ta = a.startTime != null ? a.startTime : "";
String tb = b.startTime != null ? b.startTime : "";
return tb.compareTo(ta);
});
if (limit > 0 && entries.size() > limit) {
return entries.subList(0, limit);
}
return entries;
}
public void initializeProjectEnvironment(AutomationCallback callback) throws IOException {
Files.createDirectories(baseDir);
Files.createDirectories(pythonDir);
@@ -113,6 +181,56 @@ public class MobileAutomationService {
callback.onEvent("log", "INFO", "Python 依赖安装完成");
}
/**
* 安装辅助截图工具（minicap/yosemite）到指定设备
*/
public void installAssistTools(String deviceSerial, AutomationCallback callback) throws IOException {
Path python = resolvePythonExecutable();
if (!Files.exists(python)) {
throw new IOException("Python 环境未初始化，请先点击\"初始化环境\"");
}
String deviceUri = "Android:///" + nullToEmpty(deviceSerial);
List<String> command = Arrays.asList(
python.toString(),
pythonDir.resolve("install_tools.py").toString(),
"--device", deviceUri,
"--tool", "all",
"--marker-dir", baseDir.toString()
);
callback.onEvent("log", "INFO", "正在安装辅助截图工具到设备: " + deviceSerial);
CmdResult result = runStreamingCommand(command, baseDir, 120, callback);
if (result.ok) {
callback.onEvent("log", "INFO", "辅助工具安装完成");
} else {
callback.onEvent("log", "WARN", "部分工具安装失败，运行用例时将自动回退到 ADB 截图方式");
}
}
/**
* 检查指定设备是否已安装 minicap
*/
public boolean isMinicapInstalled(String deviceSerial) {
String safeSerial = sanitizeSerial(deviceSerial);
Path marker = baseDir.resolve(".minicap-" + safeSerial);
return Files.exists(marker);
}
/**
* 检查指定设备是否已安装 Yosemite
*/
public boolean isYosemiteInstalled(String deviceSerial) {
String safeSerial = sanitizeSerial(deviceSerial);
Path marker = baseDir.resolve(".yosemite-" + safeSerial);
return Files.exists(marker);
}
private String sanitizeSerial(String serial) {
if (serial == null || serial.trim().isEmpty()) {
return "default";
}
return serial.trim().replace(":", "_").replace("/", "_");
}
public EnvironmentCheckResult checkEnvironment() {
EnvironmentCheckResult result = new EnvironmentCheckResult();
Path python = resolvePythonExecutable();
@@ -184,8 +302,11 @@ public class MobileAutomationService {
public AutomationRunHandle runCase(AutomationRunConfig config, AutomationCallback callback) throws IOException {
validateRunConfig(config);
String runId = LocalDateTime.now().format(RUN_ID_FMT);
Path reportDir = reportsDir.resolve(runId);
LocalDateTime now = LocalDateTime.now();
String runId = now.format(RUN_ID_FMT);
// 报告目录使用可读格式: 报告 2026-07-06 19-13-02
String reportDirName = "报告 " + now.format(REPORT_DIR_FMT);
Path reportDir = reportsDir.resolve(reportDirName);
Files.createDirectories(reportDir);
List<String> command = new ArrayList<>();
@@ -206,9 +327,18 @@ public class MobileAutomationService {
if (config.isScreenshotOnFail()) {
command.add("--screenshot-on-fail");
}
// 如果设备已安装辅助工具，启用高效截图模式
if (isMinicapInstalled(config.getDeviceSerial())) {
command.add("--use-minicap");
callback.onEvent("log", "INFO", "minicap 已安装，启用高效截图模式");
} else if (isYosemiteInstalled(config.getDeviceSerial())) {
command.add("--use-javacap");
callback.onEvent("log", "INFO", "Yosemite 已安装，启用 javacap 高效截图模式");
}
ProcessBuilder pb = new ProcessBuilder(command);
pb.directory(baseDir.toFile());
pb.environment().put("PYTHONIOENCODING", "utf-8");
pb.redirectErrorStream(false);
Process process = pb.start();
AutomationRunHandle handle = new AutomationRunHandle(runId, process);
@@ -253,10 +383,86 @@ public class MobileAutomationService {
}
public synchronized void startRecording(String deviceSerial, String packageName) throws IOException {
String runId = "record-" + LocalDateTime.now().format(RUN_ID_FMT);
Path recordDir = reportsDir.resolve(runId);
LocalDateTime now = LocalDateTime.now();
String runId = "record-" + now.format(RUN_ID_FMT);
// 报告目录使用可读格式: 报告 2026-07-06 19-13-02
String reportDirName = "报告 " + now.format(REPORT_DIR_FMT);
Path recordDir = reportsDir.resolve(reportDirName);
Files.createDirectories(recordDir.resolve("screenshots"));
recordingSession = new RecordingSession(runId, deviceSerial, packageName, recordDir);
// 如果未提供包名，尝试从设备获取当前前台应用
String actualPackageName = packageName;
if (actualPackageName == null || actualPackageName.trim().isEmpty()) {
actualPackageName = getCurrentForegroundApp(deviceSerial);
if (actualPackageName != null) {
log.info("自动检测到前台应用: {}", actualPackageName);
}
}
recordingSession = new RecordingSession(runId, deviceSerial, actualPackageName, recordDir, now);
}
/**
* 获取设备当前前台应用的包名（排除桌面启动器）
*/
public String getCurrentForegroundApp(String deviceSerial) {
try {
List<String> command = new ArrayList<>();
command.add(resolveAdbExecutable().orElse(Path.of("adb")).toString());
if (deviceSerial != null && !deviceSerial.trim().isEmpty()) {
command.add("-s");
command.add(deviceSerial.trim());
}
// 使用 dumpsys window 获取当前焦点窗口
command.addAll(Arrays.asList("shell", "dumpsys", "window", "|", "grep", "mCurrentFocus"));
CmdResult result = runCommand(command, null, 5);
if (result.ok && result.output != null && !result.output.isEmpty()) {
// 输出格式类似: mCurrentFocus=Window{... u0 com.example.app/com.example.app.MainActivity}
String output = result.output.trim();
java.util.regex.Pattern pattern = java.util.regex.Pattern.compile("u\\d+\\s+([a-zA-Z0-9._]+)");
java.util.regex.Matcher matcher = pattern.matcher(output);
if (matcher.find()) {
String packageName = matcher.group(1);
// 排除桌面启动器包名
if (isLauncherPackage(packageName)) {
log.info("检测到桌面启动器包名: {}，跳过自动填充", packageName);
return null;
}
return packageName;
}
}
} catch (Exception e) {
log.warn("获取前台应用失败: {}", e.getMessage());
}
return null;
}
/**
* 判断是否为桌面启动器包名
*/
private static final Set<String> LAUNCHER_PACKAGES = Set.of(
"com.miui.home", // 小米
"com.android.launcher", // AOSP
"com.android.launcher3", // AOSP
"com.sec.android.app.launcher", // 三星
"com.huawei.android.launcher", // 华为
"com.oppo.launcher", // OPPO
"com.coloros.launcher", // OPPO ColorOS
"com.bbk.launcher", // Vivo
"com.vivo.launcher", // Vivo
"com.oneplus.launcher", // 一加
"com.google.android.apps.nexuslauncher", // Pixel
"com.teslacoilsw.launcher", // Nova Launcher
"com.motorola.launcher3" // 摩托罗拉
);
private boolean isLauncherPackage(String packageName) {
if (packageName == null) return false;
String lower = packageName.toLowerCase(Locale.ROOT);
if (LAUNCHER_PACKAGES.contains(lower)) return true;
// 通配匹配：包含 launcher 关键字的包名
return lower.contains(".launcher") || lower.endsWith(".home");
}
public synchronized boolean isRecording() {
@@ -267,8 +473,13 @@ public class MobileAutomationService {
if (recordingSession == null) {
throw new IOException("尚未开始录制");
}
Path screenshot = recordingSession.recordDir.resolve("screenshots/current.png");
// 使用带时间戳的文件名避免覆盖，同时保留 current.png 作为最新截图的快捷方式
String timestamp = LocalDateTime.now().format(RUN_ID_FMT);
Path screenshot = recordingSession.recordDir.resolve("screenshots/screenshot-" + timestamp + ".png");
captureScreenshot(recordingSession.deviceSerial, screenshot);
// 同步创建 current.png 软链接（或复制），供实时显示使用
Path current = recordingSession.recordDir.resolve("screenshots/current.png");
Files.copy(screenshot, current, StandardCopyOption.REPLACE_EXISTING);
return screenshot;
}
@@ -279,7 +490,7 @@ public class MobileAutomationService {
int x = scale(imageX, imageWidth, screenWidth);
int y = scale(imageY, imageHeight, screenHeight);
runAdb(recordingSession.deviceSerial, Arrays.asList("shell", "input", "tap", String.valueOf(x), String.valueOf(y)), 10);
recordingSession.addTap(x, y, screenWidth, screenHeight);
recordingSession.addTap(x, y, screenWidth, screenHeight, defaultWaitTimeMs);
}
public synchronized void recordInput(String text) throws IOException {
@@ -290,7 +501,7 @@ public class MobileAutomationService {
text = "";
}
runAdb(recordingSession.deviceSerial, Arrays.asList("shell", "input", "text", escapeAdbInput(text)), 10);
recordingSession.addInput(text);
recordingSession.addInput(text, defaultWaitTimeMs);
}
public synchronized void recordBack() throws IOException {
@@ -298,7 +509,7 @@ public class MobileAutomationService {
throw new IOException("尚未开始录制");
}
runAdb(recordingSession.deviceSerial, Arrays.asList("shell", "input", "keyevent", "BACK"), 10);
recordingSession.addKeyevent("BACK");
recordingSession.addKeyevent("BACK", defaultWaitTimeMs);
}
public synchronized void recordSwipe(int startX, int startY, int endX, int endY, int durationMs) throws IOException {
@@ -307,22 +518,362 @@ public class MobileAutomationService {
}
runAdb(recordingSession.deviceSerial, Arrays.asList("shell", "input", "swipe",
String.valueOf(startX), String.valueOf(startY), String.valueOf(endX), String.valueOf(endY), String.valueOf(durationMs)), 10);
recordingSession.addSwipe(startX, startY, endX, endY, durationMs);
recordingSession.addSwipe(startX, startY, endX, endY, durationMs, defaultWaitTimeMs);
}
/**
* 撤销上一步录制操作
* @return 被撤销的步骤描述，null 表示没有可撤销的步骤
*/
public synchronized String undoLastStep() {
if (recordingSession == null || recordingSession.steps.isEmpty()) {
return null;
}
JSONObject removed = recordingSession.steps.remove(recordingSession.steps.size() - 1);
recordingSession.index--;
// 返回步骤名称用于显示
return removed.getString("name");
}
/**
* 更新指定步骤的等待时间
* @param index 步骤索引（从0开始）
* @param waitMs 新的等待时间（毫秒）
* @return 是否成功
*/
public synchronized boolean updateStepWaitTime(int index, int waitMs) {
if (recordingSession == null || index < 0 || index >= recordingSession.steps.size()) {
return false;
}
JSONObject step = recordingSession.steps.get(index);
step.put("waitAfterMs", waitMs);
return true;
}
/**
* 删除指定索引的步骤
* @param index 步骤索引（从0开始）
* @return 是否成功
*/
public synchronized boolean deleteStep(int index) {
if (recordingSession == null || index < 0 || index >= recordingSession.steps.size()) {
return false;
}
recordingSession.steps.remove(index);
// 重新编号后续步骤
for (int i = index; i < recordingSession.steps.size(); i++) {
JSONObject step = recordingSession.steps.get(i);
String name = step.getString("name");
if (name != null) {
// 将 tap-N、input-N 等改为正确的序号
String newName = name.replaceAll("-(\\d+)$", "-" + (i + 1));
step.put("name", newName);
}
}
recordingSession.index = recordingSession.steps.size() + 1;
return true;
}
/**
* 设置指定步骤的页面特征
* @param index 步骤索引（从0开始）
* @param featureName 页面特征名称（如"login-page"、"ad-popup"等）
* @param screenshotPath 截图路径（可选，null表示不设置截图）
* @return 是否成功
*/
public synchronized boolean setStepPageFeature(int index, String featureName, String screenshotPath) {
if (recordingSession == null || index < 0 || index >= recordingSession.steps.size()) {
return false;
}
JSONObject step = recordingSession.steps.get(index);
if (featureName == null || featureName.trim().isEmpty()) {
step.remove("pageFeature");
step.remove("pageFeatureScreenshot");
} else {
step.put("pageFeature", featureName.trim());
if (screenshotPath != null && !screenshotPath.isEmpty()) {
step.put("pageFeatureScreenshot", screenshotPath);
}
}
return true;
}
/**
* 获取指定步骤的页面特征信息
* @param index 步骤索引（从0开始）
* @return 页面特征名称，如果没有则返回 null
*/
public synchronized String getStepPageFeature(int index) {
if (recordingSession == null || index < 0 || index >= recordingSession.steps.size()) {
return null;
}
JSONObject step = recordingSession.steps.get(index);
return step.getString("pageFeature");
}
/**
* 获取指定步骤的完整JSON信息
* @param index 步骤索引（从0开始）
* @return 步骤的 JSONObject，如果无效则返回 null
*/
public synchronized JSONObject getStepInfo(int index) {
if (recordingSession == null || index < 0 || index >= recordingSession.steps.size()) {
return null;
}
return recordingSession.steps.get(index);
}
/**
* 获取设备当前页面的 UI 层级信息
* @param deviceSerial 设备序列号
* @return UI 元素列表，每个元素包含 resourceId, text, contentDesc, className, bounds
*/
public List<JSONObject> dumpUiHierarchy(String deviceSerial) throws IOException {
// 1. 在设备上生成 UI dump
Path localDump = baseDir.resolve("window_dump.xml");
runAdb(deviceSerial, Arrays.asList("shell", "uiautomator", "dump", "/sdcard/window_dump.xml"), 15);
// 2. 拉取到本地
runAdb(deviceSerial, Arrays.asList("pull", "/sdcard/window_dump.xml", localDump.toString()), 10);
// 3. 解析 XML
List<JSONObject> elements = new ArrayList<>();
if (!Files.exists(localDump)) {
throw new IOException("UI dump 文件不存在");
}
try {
String xmlContent = new String(Files.readAllBytes(localDump), StandardCharsets.UTF_8);
// 简单解析 XML 中的 node 元素
javax.xml.parsers.DocumentBuilderFactory factory = javax.xml.parsers.DocumentBuilderFactory.newInstance();
javax.xml.parsers.DocumentBuilder builder = factory.newDocumentBuilder();
org.w3c.dom.Document doc = builder.parse(new ByteArrayInputStream(xmlContent.getBytes(StandardCharsets.UTF_8)));
org.w3c.dom.NodeList nodes = doc.getElementsByTagName("node");
for (int i = 0; i < nodes.getLength(); i++) {
org.w3c.dom.Element node = (org.w3c.dom.Element) nodes.item(i);
JSONObject elem = new JSONObject();
elem.put("resourceId", node.getAttribute("resource-id"));
elem.put("text", node.getAttribute("text"));
elem.put("contentDesc", node.getAttribute("content-desc"));
elem.put("className", node.getAttribute("class"));
elem.put("bounds", node.getAttribute("bounds"));
elem.put("clickable", "true".equals(node.getAttribute("clickable")));
elem.put("enabled", "true".equals(node.getAttribute("enabled")));
// 解析 bounds: [left,top][right,bottom]
String bounds = node.getAttribute("bounds");
if (bounds != null && !bounds.isEmpty()) {
java.util.regex.Matcher m = java.util.regex.Pattern.compile("\\[(\\d+),(\\d+)\\]\\[(\\d+),(\\d+)\\]").matcher(bounds);
if (m.find()) {
elem.put("left", Integer.parseInt(m.group(1)));
elem.put("top", Integer.parseInt(m.group(2)));
elem.put("right", Integer.parseInt(m.group(3)));
elem.put("bottom", Integer.parseInt(m.group(4)));
}
}
elements.add(elem);
}
} catch (javax.xml.parsers.ParserConfigurationException | org.xml.sax.SAXException e) {
throw new IOException("解析 UI dump XML 失败: " + e.getMessage());
} finally {
try { Files.deleteIfExists(localDump); } catch (IOException ignored) {}
}
return elements;
}
/**
* 识别指定坐标处的 UI 元素
* @param deviceSerial 设备序列号
* @param x 屏幕 X 坐标
* @param y 屏幕 Y 坐标
* @return 匹配的元素列表（按优先级排序：resource-id > content-desc > text）
*/
public List<JSONObject> identifyElement(String deviceSerial, int x, int y) throws IOException {
List<JSONObject> allElements = dumpUiHierarchy(deviceSerial);
List<JSONObject> matched = new ArrayList<>();
for (JSONObject elem : allElements) {
Integer left = elem.getInteger("left");
Integer top = elem.getInteger("top");
Integer right = elem.getInteger("right");
Integer bottom = elem.getInteger("bottom");
if (left != null && top != null && right != null && bottom != null) {
if (x >= left && x <= right && y >= top && y <= bottom) {
// 计算面积用于排序（优先选择更精确的小元素）
int area = (right - left) * (bottom - top);
elem.put("area", area);
matched.add(elem);
}
}
}
// 按面积升序排序（面积越小越精确）
matched.sort((a, b) -> a.getIntValue("area") - b.getIntValue("area"));
return matched;
}
/**
* 更新指定步骤的目标元素信息（元素优先+坐标兑底）
* @param index 步骤索引
* @param element 识别到的元素信息（来自 identifyElement）
* @return 是否成功
*/
public synchronized boolean updateStepElement(int index, JSONObject element) {
if (recordingSession == null || index < 0 || index >= recordingSession.steps.size()) {
return false;
}
JSONObject step = recordingSession.steps.get(index);
JSONObject target = step.getJSONObject("target");
if (target == null) {
target = new JSONObject();
step.put("target", target);
}
// 设置 Poco 查询信息
JSONObject poco = new JSONObject();
String resourceId = element.getString("resourceId");
String contentDesc = element.getString("contentDesc");
String text = element.getString("text");
if (resourceId != null && !resourceId.isEmpty()) {
poco.put("name", resourceId);
} else if (contentDesc != null && !contentDesc.isEmpty()) {
poco.put("name", contentDesc);
} else if (text != null && !text.isEmpty()) {
poco.put("text", text);
} else {
return false; // 无可识别属性
}
target.put("preferred", "poco");
target.put("poco", poco);
return true;
}
/**
* 更新已保存用例文件中的步骤元素信息
* @param caseFile 用例文件路径
* @param stepIndex 步骤索引
* @param element 识别到的元素信息
* @return 是否成功
*/
public boolean updateCaseStepElement(Path caseFile, int stepIndex, JSONObject element) throws IOException {
if (caseFile == null || !Files.exists(caseFile)) {
return false;
}
String content = new String(Files.readAllBytes(caseFile), StandardCharsets.UTF_8);
JSONObject caseJson = JSON.parseObject(content);
com.alibaba.fastjson.JSONArray steps = caseJson.getJSONArray("steps");
if (steps == null || stepIndex < 0 || stepIndex >= steps.size()) {
return false;
}
JSONObject step = steps.getJSONObject(stepIndex);
JSONObject target = step.getJSONObject("target");
if (target == null) {
target = new JSONObject();
step.put("target", target);
}
JSONObject poco = new JSONObject();
String resourceId = element.getString("resourceId");
String contentDesc = element.getString("contentDesc");
String text = element.getString("text");
if (resourceId != null && !resourceId.isEmpty()) {
poco.put("name", resourceId);
} else if (contentDesc != null && !contentDesc.isEmpty()) {
poco.put("name", contentDesc);
} else if (text != null && !text.isEmpty()) {
poco.put("text", text);
} else {
return false;
}
target.put("preferred", "poco");
target.put("poco", poco);
Files.write(caseFile, caseJson.toJSONString().getBytes(StandardCharsets.UTF_8));
return true;
}
/**
* 对已保存的用例文件进行 UI dump 并识别指定步骤的元素
* @param deviceSerial 设备序列号
* @param caseFile 用例文件
* @param stepIndex 步骤索引
* @return 匹配的元素列表
*/
public List<JSONObject> identifyCaseStepElement(String deviceSerial, Path caseFile, int stepIndex) throws IOException {
if (caseFile == null || !Files.exists(caseFile)) {
throw new IOException("用例文件不存在");
}
String content = new String(Files.readAllBytes(caseFile), StandardCharsets.UTF_8);
JSONObject caseJson = JSON.parseObject(content);
com.alibaba.fastjson.JSONArray steps = caseJson.getJSONArray("steps");
if (steps == null || stepIndex < 0 || stepIndex >= steps.size()) {
throw new IOException("步骤索引无效");
}
JSONObject step = steps.getJSONObject(stepIndex);
JSONObject target = step.getJSONObject("target");
if (target == null || !"tap".equals(step.getString("action"))) {
return Collections.emptyList();
}
int x = target.getIntValue("x");
int y = target.getIntValue("y");
return identifyElement(deviceSerial, x, y);
}
public synchronized Path stopRecording() throws IOException {
if (recordingSession == null) {
throw new IOException("尚未开始录制");
}
Path file = casesDir.resolve(recordingSession.runId + ".yaml");
Files.write(file, recordingSession.toYaml().getBytes(StandardCharsets.UTF_8));
Path latest = casesDir.resolve("latest-recorded-flow.yaml");
// 生成可读用例文件名: 2026-07-06 19-13 用例（N步）.json
int stepCount = recordingSession.steps.size();
String caseName = recordingSession.startTime.format(CASE_NAME_FMT) + " 用例（" + stepCount + "步）";
Path file = casesDir.resolve(caseName + ".json");
// 更新 JSON 内的 name 字段
recordingSession.caseName = caseName;
Files.write(file, recordingSession.toJSON().getBytes(StandardCharsets.UTF_8));
Path latest = casesDir.resolve("latest-recorded-flow.json");
Files.copy(file, latest, StandardCopyOption.REPLACE_EXISTING);
recordingSession = null;
return file;
}
private void captureScreenshot(String serial, Path output) throws IOException {
/**
* 重命名用例文件
*/
public Path renameCase(Path oldFile, String newName) throws IOException {
if (oldFile == null || !Files.exists(oldFile)) {
throw new IOException("文件不存在");
}
// 清理文件名中的非法字符
String safeName = newName.replaceAll("[\\\\/:*?\"<>|]", "-");
Path newFile = oldFile.getParent().resolve(safeName + ".json");
if (Files.exists(newFile) && !newFile.equals(oldFile)) {
throw new IOException("文件名已存在: " + safeName);
}
// 更新 JSON 内部的 name 字段
try {
String content = new String(Files.readAllBytes(oldFile), StandardCharsets.UTF_8);
JSONObject json = JSON.parseObject(content);
json.put("name", safeName);
Files.write(oldFile, json.toJSONString().getBytes(StandardCharsets.UTF_8));
} catch (Exception e) {
log.warn("更新用例name字段失败: {}", e.getMessage());
}
if (!newFile.equals(oldFile)) {
Files.move(oldFile, newFile, StandardCopyOption.REPLACE_EXISTING);
}
return newFile;
}
/**
* 重命名报告目录
*/
public Path renameReport(Path oldDir, String newName) throws IOException {
if (oldDir == null || !Files.isDirectory(oldDir)) {
throw new IOException("报告目录不存在");
}
String safeName = newName.replaceAll("[\\\\/:*?\"<>|]", "-");
Path newDir = oldDir.getParent().resolve(safeName);
if (Files.exists(newDir) && !newDir.equals(oldDir)) {
throw new IOException("目录名已存在: " + safeName);
}
if (!newDir.equals(oldDir)) {
Files.move(oldDir, newDir, StandardCopyOption.REPLACE_EXISTING);
}
return newDir;
}
public void captureScreenshot(String serial, Path output) throws IOException {
List<String> command = new ArrayList<>();
command.add(resolveAdbExecutable().orElse(Path.of("adb")).toString());
if (serial != null && !serial.trim().isEmpty()) {
@@ -364,7 +915,7 @@ public class MobileAutomationService {
private boolean isCaseFile(Path path) {
String name = path.getFileName().toString().toLowerCase(Locale.ROOT);
return name.endsWith(".yaml") || name.endsWith(".yml") || name.endsWith(".py") || name.endsWith(".air");
return name.endsWith(".json") || name.endsWith(".yaml") || name.endsWith(".yml") || name.endsWith(".py") || name.endsWith(".air");
}
private void addCheck(EnvironmentCheckResult result, String name, List<String> command) {
@@ -388,6 +939,19 @@ public class MobileAutomationService {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8))) {
String line;
while ((line = reader.readLine()) != null) {
// 过滤 Airtest 内部 DEBUG 日志（Airtest 将 DEBUG 日志写到 stderr）
if (line.contains("[DEBUG]") || line.contains("[DEBUG]<")) {
continue; // 跳过 DEBUG 级别日志
}
// 过滤 Airtest 内部的 INFO 日志（也写到 stderr）
if (line.matches(".*\\[INFO\\]<airtest\\..*")) {
continue; // 跳过 Airtest 内部 INFO 日志
}
// 过滤 Python warnings
if (line.contains("UserWarning") || line.contains("warnings.warn")) {
continue;
}
// 只有真正的错误才显示
callback.onEvent("log", "ERROR", line);
}
} catch (IOException e) {
@@ -413,6 +977,20 @@ public class MobileAutomationService {
}
}
/**
* 检查指定设备是否仍然在线
*/
public boolean isDeviceConnected(String serial) {
try {
Optional<Path> adb = resolveAdbExecutable();
if (adb.isEmpty()) return false;
CmdResult result = runCommand(Arrays.asList(adb.get().toString(), "-s", serial, "get-state"), null, 5);
return result.ok && result.output.trim().equals("device");
} catch (Exception e) {
return false;
}
}
private void runAdb(String serial, List<String> args, int timeoutSeconds) throws IOException {
CmdResult result = runCommand(buildAdbCommand(serial, args), null, timeoutSeconds);
if (!result.ok) {
@@ -476,6 +1054,8 @@ public class MobileAutomationService {
if (workingDir != null) {
pb.directory(workingDir.toFile());
}
// 确保 Python 输出使用 UTF-8 编码，避免中文乱码
pb.environment().put("PYTHONIOENCODING", "utf-8");
pb.redirectErrorStream(true);
process = pb.start();
Process proc = process;
@@ -679,73 +1259,113 @@ public class MobileAutomationService {
final String deviceSerial;
final String packageName;
final Path recordDir;
final List<String> steps = new ArrayList<>();
final LocalDateTime startTime;
String caseName; // 停止录制时设置
final List<JSONObject> steps = new ArrayList<>();
int index = 1;
RecordingSession(String runId, String deviceSerial, String packageName, Path recordDir) {
RecordingSession(String runId, String deviceSerial, String packageName, Path recordDir, LocalDateTime startTime) {
this.runId = runId;
this.deviceSerial = deviceSerial;
this.packageName = packageName;
this.recordDir = recordDir;
this.startTime = startTime;
this.caseName = runId; // 默认值
}
void addTap(int x, int y, int screenWidth, int screenHeight) {
steps.add(" - name: tap-" + index + "\n"
+ " action: tap\n"
+ " target:\n"
+ " type: coord\n"
+ " preferred: coord\n"
+ " x: " + x + "\n"
+ " y: " + y + "\n"
+ " screenWidth: " + screenWidth + "\n"
+ " screenHeight: " + screenHeight + "\n"
+ " waitAfterMs: 500");
void addTap(int x, int y, int screenWidth, int screenHeight, int waitAfterMs) {
JSONObject step = new JSONObject();
step.put("name", "tap-" + index);
step.put("action", "tap");
JSONObject target = new JSONObject();
target.put("type", "coord");
target.put("preferred", "coord");
target.put("x", x);
target.put("y", y);
target.put("screenWidth", screenWidth);
target.put("screenHeight", screenHeight);
step.put("target", target);
step.put("waitAfterMs", waitAfterMs);
// 可选：页面特征（用于执行时验证当前页面）
// step.put("pageFeature", "login-page");
steps.add(step);
index++;
}
void addInput(String text) {
steps.add(" - name: input-" + index + "\n"
+ " action: input_text\n"
+ " text: \"" + yamlEscape(text) + "\"\n"
+ " waitAfterMs: 500");
void addInput(String text, int waitAfterMs) {
JSONObject step = new JSONObject();
step.put("name", "input-" + index);
step.put("action", "input_text");
step.put("text", text);
step.put("waitAfterMs", waitAfterMs);
steps.add(step);
index++;
}
void addKeyevent(String key) {
steps.add(" - name: key-" + index + "\n"
+ " action: keyevent\n"
+ " key: " + key + "\n"
+ " waitAfterMs: 500");
void addKeyevent(String key, int waitAfterMs) {
JSONObject step = new JSONObject();
step.put("name", "key-" + index);
step.put("action", "keyevent");
step.put("key", key);
step.put("waitAfterMs", waitAfterMs);
steps.add(step);
index++;
}
void addSwipe(int startX, int startY, int endX, int endY, int durationMs) {
steps.add(" - name: swipe-" + index + "\n"
+ " action: swipe\n"
+ " start: {x: " + startX + ", y: " + startY + "}\n"
+ " end: {x: " + endX + ", y: " + endY + "}\n"
+ " durationMs: " + durationMs + "\n"
+ " waitAfterMs: 500");
void addSwipe(int startX, int startY, int endX, int endY, int durationMs, int waitAfterMs) {
JSONObject step = new JSONObject();
step.put("name", "swipe-" + index);
step.put("action", "swipe");
JSONObject start = new JSONObject();
start.put("x", startX);
start.put("y", startY);
step.put("start", start);
JSONObject end = new JSONObject();
end.put("x", endX);
end.put("y", endY);
step.put("end", end);
step.put("durationMs", durationMs);
step.put("waitAfterMs", waitAfterMs);
steps.add(step);
index++;
}
String toYaml() {
StringBuilder yaml = new StringBuilder();
yaml.append("name: ").append(runId).append("\n");
yaml.append("app:\n");
yaml.append(" package: ").append(packageName == null ? "" : packageName).append("\n");
yaml.append(" activity: \n");
yaml.append("device:\n");
yaml.append(" connect: Android:///");
yaml.append(deviceSerial == null ? "" : deviceSerial).append("\n");
yaml.append("settings:\n");
yaml.append(" timeout: 20\n");
yaml.append(" screenshotOnFail: true\n");
yaml.append("steps:\n");
for (String step : steps) {
yaml.append(step).append("\n");
}
return yaml.toString();
/**
* 生成 JSON 格式的用例文件
*/
String toJSON() {
JSONObject caseJson = new JSONObject();
caseJson.put("name", caseName);
// App 信息
JSONObject app = new JSONObject();
app.put("package", packageName == null ? "" : packageName);
app.put("activity", "");
caseJson.put("app", app);
// 设备信息
JSONObject device = new JSONObject();
device.put("connect", "Android:///" + (deviceSerial == null ? "" : deviceSerial));
caseJson.put("device", device);
// 设置
JSONObject settings = new JSONObject();
settings.put("timeout", 20);
settings.put("screenshotOnFail", true);
caseJson.put("settings", settings);
// 步骤列表
caseJson.put("steps", steps);
return caseJson.toJSONString();
}
private String yamlEscape(String text) {
src/main/java/com/example/workutils/biz/mobileuiauto/service/model/ExecutionHistoryEntry.java
+36
View File
@@ -0,0 +1,36 @@
package com.example.workutils.biz.mobileuiauto.service.model;
import java.nio.file.Path;
import java.util.List;
/**
* 执行历史记录条目
*/
public class ExecutionHistoryEntry {
/** 报告目录 */
public Path reportDir;
/** 用例名称 */
public String caseName;
/** 设备标识 */
public String device;
/** 包名 */
public String packageName;
/** 是否成功 */
public boolean success;
/** 开始时间（ISO 格式字符串） */
public String startTime;
/** 结束时间（ISO 格式字符串） */
public String endTime;
/** 步骤列表 */
public List<Object> steps;
@Override
public String toString() {
String icon = success ? "✅" : "❌";
String time = startTime != null && startTime.length() >= 16
? startTime.substring(0, 16).replace("T", " ")
: startTime;
int stepCount = steps != null ? steps.size() : 0;
return icon + " " + caseName + " (" + stepCount + "步) " + time;
}
}
src/main/resources/css/light-theme.css
+15
View File
@@ -291,3 +291,18 @@
-fx-padding: 8 0;
-fx-alignment: CENTER_LEFT;
}
/* ---- 紧凑列表单元格 ---- */
.compact-list .list-cell {
-fx-padding: 3 8 3 8;
-fx-font-size: 13;
}
.compact-list .list-cell:selected {
-fx-background-color: derive(-workutils-primary, 85%);
-fx-text-fill: -workutils-text;
}
.compact-list .list-cell:filled:hover {
-fx-background-color: -workutils-card;
}
src/main/resources/mobile-ui-auto/python/install_tools.py
+286
View File
@@ -0,0 +1,286 @@
"""安装 Airtest 辅助工具（minicap/yosemite）到设备"""
import argparse
import json
import os
import subprocess
import sys
import time
def emit(event_type, level="INFO", message="", **extra):
payload = {"type": event_type, "level": level, "message": message}
payload.update(extra)
print(json.dumps(payload, ensure_ascii=False), flush=True)
def find_adb():
"""查找 adb 可执行文件"""
venv_dir = os.path.join(os.path.dirname(__file__), "..", ".venv")
candidates = [
os.path.join(venv_dir, "Lib", "site-packages", "airtest", "core", "android", "static", "adb", "windows", "adb.exe"),
os.path.join(venv_dir, "lib", "python3.11", "site-packages", "airtest", "core", "android", "static", "adb", "linux", "adb"),
os.path.join(venv_dir, "lib", "python3.10", "site-packages", "airtest", "core", "android", "static", "adb", "linux", "adb"),
]
for c in candidates:
if os.path.exists(c):
return c
return "adb"
def install_minicap(device_uri):
"""通过 Airtest 安装 minicap 并验证"""
try:
from airtest.core.api import auto_setup
import logging
logging.getLogger("airtest").setLevel(logging.INFO)
# 提取 serial
serial = device_uri.replace("Android:///", "").strip()
serial = serial.split("?")[0] if "?" in serial else serial
emit("log", "INFO", "正在连接设备并安装 minicap...")
# 不使用 cap_method=adbcap，让 Airtest 尝试安装 minicap
auto_setup(__file__, devices=[device_uri])
# 等待安装完成
time.sleep(3)
# 通过 ADB 检查 minicap 二进制文件是否存在于设备上
adb = find_adb()
check_cmd = [adb]
if serial:
check_cmd.extend(["-s", serial])
check_cmd.extend(["shell", "ls", "/data/local/tmp/minicap"])
emit("log", "INFO", "检查 minicap 是否已安装到设备...")
result = subprocess.run(check_cmd, capture_output=True, text=True, timeout=10)
combined = (result.stdout + result.stderr).strip()
if "No such file" in combined or "not found" in combined.lower():
emit("log", "WARN", "minicap 二进制文件未找到，安装失败")
return False
# minicap 文件存在，进一步验证是否能正常运行
# 尝试启动 minicap 并立即停止
test_cmd = [adb]
if serial:
test_cmd.extend(["-s", serial])
test_cmd.extend(["shell", "LD_LIBRARY_PATH=/data/local/tmp", "/data/local/tmp/minicap", "-i"])
emit("log", "INFO", "验证 minicap 是否可正常运行...")
try:
test_result = subprocess.run(test_cmd, capture_output=True, text=True, timeout=5)
test_output = (test_result.stdout + test_result.stderr).strip()
if "pid=" in test_output or "web socket" in test_output.lower() or result.returncode == 0:
emit("log", "INFO", "minicap 安装成功且可正常运行")
return True
else:
emit("log", "WARN", f"minicap 运行测试输出: {test_output}")
# 即使测试输出不明确，文件存在也算安装成功
return True
except subprocess.TimeoutExpired:
# minicap 作为服务运行会超时，这是正常的
emit("log", "INFO", "minicap 安装成功（服务进程已启动）")
return True
except Exception as e:
emit("log", "ERROR", f"安装 minicap 失败: {e}")
return False
def install_yosemite(device_uri):
"""安装 Yosemite APK（用于 javacap 截图）"""
try:
from airtest.core.api import auto_setup, connect_device
import logging
logging.getLogger("airtest").setLevel(logging.INFO)
serial = device_uri.replace("Android:///", "").strip()
serial = serial.split("?")[0] if "?" in serial else serial
emit("log", "INFO", "正在连接设备...")
auto_setup(__file__, devices=[device_uri])
time.sleep(1)
# 通过 Airtest 的 Android 设备对象直接安装 Yosemite
from airtest.core.helper import G
device = G.DEVICE
emit("log", "INFO", "正在安装 Yosemite APK...")
# 使用设备对象的 yosemite 安装方法
try:
from airtest.core.android.yosemite import Yosemite
yosemite = Yosemite(device.adb)
yosemite.install_or_upgrade()
emit("log", "INFO", "Yosemite 安装成功")
return True
except TypeError:
# 旧版本 API 兼容
from airtest.core.android.yosemite import Yosemite
yosemite = Yosemite(device)
yosemite.install_or_upgrade()
emit("log", "INFO", "Yosemite 安装成功")
return True
except Exception as e:
emit("log", "WARN", f"Yosemite 安装失败: {e}")
return False
def main():
parser = argparse.ArgumentParser(description="安装 Airtest 辅助工具")
parser.add_argument("--device", default="Android:///")
parser.add_argument("--tool", default="minicap", choices=["minicap", "yosemite", "all"])
parser.add_argument("--marker-dir", required=True, help="标记文件存放目录")
args = parser.parse_args()
os.makedirs(args.marker_dir, exist_ok=True)
# 从设备 URI 中提取 serial 用于标记文件名
serial = args.device.replace("Android:///", "").strip()
serial = serial.split("?")[0] if "?" in serial else serial
serial = serial if serial else "default"
safe_serial = serial.replace(":", "_").replace("/", "_")
results = {"minicap": False, "yosemite": False}
if args.tool in ("minicap", "all"):
results["minicap"] = install_minicap(args.device)
if args.tool in ("yosemite", "all"):
results["yosemite"] = install_yosemite(args.device)
# 写入标记文件
for tool_name, success in results.items():
marker_file = os.path.join(args.marker_dir, f".{tool_name}-{safe_serial}")
if success:
with open(marker_file, "w") as f:
f.write(json.dumps({"installed": True, "time": time.strftime("%Y-%m-%d %H:%M:%S")}))
emit("log", "INFO", f"{tool_name} 安装成功，已写入标记文件")
else:
# 如果失败，删除已有标记文件
if os.path.exists(marker_file):
os.remove(marker_file)
emit("log", "WARN", f"{tool_name} 安装失败，已清除标记")
# 输出最终结果
any_success = any(results.values())
emit("result", "INFO" if any_success else "ERROR",
json.dumps(results, ensure_ascii=False),
success=any_success)
return 0 if any_success else 1
if __name__ == "__main__":
sys.exit(main())
"""安装 Airtest 辅助工具（minicap/yosemite）到设备"""
import argparse
import json
import os
import sys
import time
def emit(event_type, level="INFO", message="", **extra):
payload = {"type": event_type, "level": level, "message": message}
payload.update(extra)
print(json.dumps(payload, ensure_ascii=False), flush=True)
def install_minicap(device_uri):
"""通过 Airtest 安装 minicap 并验证"""
try:
from airtest.core.api import auto_setup, snapshot
from airtest.core.android import Android as AndroidDevice
import logging
logging.getLogger("airtest").setLevel(logging.INFO)
emit("log", "INFO", "正在连接设备...")
# 不使用 cap_method=adbcap，让 Airtest 尝试安装 minicap
auto_setup(__file__, devices=[device_uri])
emit("log", "INFO", "设备已连接，正在初始化截图组件...")
# Airtest 在 auto_setup 时会自动安装 minicap，
# 尝试截图来验证是否安装成功
time.sleep(2)
test_path = os.path.join(os.path.dirname(__file__), "..", "test_minicap.png")
try:
snapshot(filename=test_path)
if os.path.exists(test_path) and os.path.getsize(test_path) > 100:
os.remove(test_path)
emit("log", "INFO", "minicap 安装成功，截图验证通过")
return True
else:
emit("log", "WARN", "截图文件过小，minicap 可能未正确安装")
if os.path.exists(test_path):
os.remove(test_path)
return False
except Exception as e:
emit("log", "WARN", f"minicap 截图测试失败: {e}")
return False
except Exception as e:
emit("log", "ERROR", f"安装 minicap 失败: {e}")
return False
def install_yosemite(device_uri):
"""安装 Yosemite APK（用于 javacap 截图）"""
try:
from airtest.core.api import auto_setup
from airtest.core.android.yosemite import Yosemite
import logging
logging.getLogger("airtest").setLevel(logging.INFO)
emit("log", "INFO", "正在安装 Yosemite APK...")
auto_setup(__file__, devices=[device_uri])
yosemite = Yosemite()
yosemite.install_or_upgrade()
emit("log", "INFO", "Yosemite 安装成功")
return True
except Exception as e:
emit("log", "WARN", f"Yosemite 安装失败: {e}")
return False
def main():
parser = argparse.ArgumentParser(description="安装 Airtest 辅助工具")
parser.add_argument("--device", default="Android:///")
parser.add_argument("--tool", default="minicap", choices=["minicap", "yosemite", "all"])
parser.add_argument("--marker-dir", required=True, help="标记文件存放目录")
args = parser.parse_args()
os.makedirs(args.marker_dir, exist_ok=True)
# 从设备 URI 中提取 serial 用于标记文件名
serial = args.device.replace("Android:///", "").strip()
serial = serial.split("?")[0] if "?" in serial else serial
serial = serial if serial else "default"
safe_serial = serial.replace(":", "_").replace("/", "_")
results = {"minicap": False, "yosemite": False}
if args.tool in ("minicap", "all"):
results["minicap"] = install_minicap(args.device)
if args.tool in ("yosemite", "all"):
results["yosemite"] = install_yosemite(args.device)
# 写入标记文件
for tool_name, success in results.items():
marker_file = os.path.join(args.marker_dir, f".{tool_name}-{safe_serial}")
if success:
with open(marker_file, "w") as f:
f.write(json.dumps({"installed": True, "time": time.strftime("%Y-%m-%d %H:%M:%S")}))
emit("log", "INFO", f"{tool_name} 安装成功，已写入标记文件")
else:
# 如果失败，删除已有标记文件
if os.path.exists(marker_file):
os.remove(marker_file)
# 输出最终结果
any_success = any(results.values())
emit("result", "INFO" if any_success else "ERROR",
json.dumps(results, ensure_ascii=False),
success=any_success)
return 0 if any_success else 1
if __name__ == "__main__":
sys.exit(main())
src/main/resources/mobile-ui-auto/python/runner.py
+114 -13
View File
@@ -14,13 +14,26 @@ def emit(event_type, level="INFO", message="", **extra):
def load_yaml(path):
"""加载 YAML 或 JSON 格式的用例文件"""
import json
try:
import yaml
has_yaml = True
except Exception as exc:
has_yaml = False
# 尝试 JSON 格式（优先）
if path.endswith('.json'):
with open(path, "r", encoding="utf-8") as fh:
return json.load(fh) or {}
# 尝试 YAML 格式
if has_yaml:
with open(path, "r", encoding="utf-8") as fh:
return yaml.safe_load(fh) or {}
else:
emit("log", "ERROR", "缺少 PyYAML，请执行: python -m pip install -U pyyaml")
raise exc
with open(path, "r", encoding="utf-8") as fh:
return yaml.safe_load(fh) or {}
raise ImportError("PyYAML not installed")
def ensure_airtest():
@@ -80,13 +93,31 @@ def run_air_case(case_path, args):
def connect(api, args):
os.makedirs(args.report, exist_ok=True)
api["auto_setup"](__file__, logdir=args.report, devices=[args.device])
# 抑制 Airtest 内部日志输出到 stderr（避免 DEBUG 日志污染输出）
import logging
logging.getLogger("airtest").setLevel(logging.WARNING)
logging.getLogger("poco").setLevel(logging.WARNING)
# 构造带参数的设备 URI
device_uri = args.device
use_minicap = getattr(args, "use_minicap", False)
use_javacap = getattr(args, "use_javacap", False)
if device_uri.startswith("Android:///") and "cap_method" not in device_uri:
sep = "?" if "?" not in device_uri else "&"
if use_minicap:
# minicap 已安装，使用默认截图方式（Airtest 自动选择 minicap）
device_uri = device_uri + sep + "touch_method=maxtouch"
emit("log", "INFO", "使用 minicap 高效截图模式")
elif use_javacap:
# Yosemite 已安装，使用 javacap 截图
device_uri = device_uri + sep + "cap_method=javacap&touch_method=maxtouch"
emit("log", "INFO", "使用 javacap (Yosemite) 高效截图模式")
else:
# 未安装辅助工具，强制使用 adbcap 避免安装尝试
device_uri = device_uri + sep + "cap_method=adbcap&touch_method=maxtouch"
emit("log", "INFO", "使用 ADB 截图模式（可通过\"安装辅助工具\"启用高效截图）")
api["auto_setup"](__file__, logdir=args.report, devices=[device_uri])
poco = None
try:
poco = api["AndroidUiautomationPoco"](use_airtest_input=True, screenshot_each_action=False)
emit("log", "INFO", "Poco 初始化成功")
except Exception as exc:
emit("log", "WARN", "Poco 初始化失败，将仅使用 Airtest/坐标/图片定位: " + str(exc))
emit("log", "INFO", "纯坐标模式运行，跳过 Poco 初始化")
return poco
@@ -161,6 +192,48 @@ def assert_exists(api, poco, target, case_dir):
raise AssertionError("图片不存在: " + target["path"])
def verify_page_feature(api, step, case_dir):
"""验证当前页面特征，返回 (matched, message)"""
page_feature = step.get("pageFeature")
if not page_feature:
return True, None
screenshot_path = step.get("pageFeatureScreenshot")
if screenshot_path:
# 有特征截图，尝试比对
try:
import cv2
import numpy as np
# 获取当前截图
current_screenshot = os.path.join(case_dir, f"current_verify_{int(time.time()*1000)}.png")
api["snapshot"](filename=current_screenshot)
# 加载特征截图
feature_img = cv2.imread(os.path.join(case_dir, screenshot_path))
current_img = cv2.imread(current_screenshot)
if feature_img is not None and current_img is not None:
# 调整大小一致
h, w = feature_img.shape[:2]
current_img = cv2.resize(current_img, (w, h))
# 计算结构相似性
diff = cv2.absdiff(feature_img, current_img)
mean_diff = np.mean(diff)
similarity = 1 - (mean_diff / 255.0)
os.remove(current_screenshot)
if similarity > 0.85:
return True, f"页面特征 '{page_feature}' 匹配 (相似度: {similarity:.2f})"
else:
return False, f"页面特征 '{page_feature}' 不匹配 (相似度: {similarity:.2f})"
os.remove(current_screenshot)
except ImportError:
pass
except Exception as e:
emit("log", "WARN", f"页面特征比对失败: {e}")
# 无特征截图或比对失败，仅记录日志
emit("log", "INFO", f"页面特征: {page_feature}")
return True, f"页面特征 '{page_feature}' 已记录（无截图比对）"
def run_yaml_case(case_path, args):
api = ensure_airtest()
case = load_yaml(case_path)
@@ -180,12 +253,29 @@ def run_yaml_case(case_path, args):
"steps": [],
}
# 常见桌面启动器包名，start_app 对启动器会失败
LAUNCHER_PACKAGES = {
"com.miui.home", "com.android.launcher", "com.android.launcher2",
"com.android.launcher3", "com.sec.android.app.launcher",
"com.huawei.android.launcher", "com.oppo.launcher",
"com.coloros.oppolauncher", "com.vivo.launcher",
"com.oneplus.launcher", "com.google.android.apps.nexuslauncher",
"com.google.android.launcher", "com.teslacoilsw.launcher",
"com.microsoft.launcher",
}
if package_name:
emit("log", "INFO", "启动 APP: " + package_name)
if activity_name:
api["start_app"](package_name, activity=activity_name)
if package_name.lower() in LAUNCHER_PACKAGES:
emit("log", "WARN", f"{package_name} 是桌面启动器，跳过启动（请手动打开被测应用）")
else:
api["start_app"](package_name)
emit("log", "INFO", "启动 APP: " + package_name)
try:
if activity_name:
api["start_app"](package_name, activity=activity_name)
else:
api["start_app"](package_name)
except Exception as e:
emit("log", "WARN", f"启动应用失败: {e}，继续执行用例")
for idx, step in enumerate(steps, start=1):
name = step.get("name") or f"step-{idx}"
@@ -194,6 +284,15 @@ def run_yaml_case(case_path, args):
emit("step_start", "INFO", f"{idx}. {name}", index=idx, name=name)
record = {"index": idx, "name": name, "action": action, "status": "PASS"}
try:
# 页面特征验证（如果步骤定义了 pageFeature）
matched, feature_msg = verify_page_feature(api, step, case_dir)
if not matched:
emit("log", "WARN", f"步骤 {idx} 页面特征不匹配: {feature_msg}")
# 页面特征不匹配时，可以选择跳过或继续执行
# 这里选择继续执行，但记录警告
elif feature_msg:
emit("log", "INFO", feature_msg)
if action in ("tap", "click"):
tap_target(api, poco, step.get("target"), case_dir)
elif action in ("input", "input_text"):
@@ -269,6 +368,8 @@ def main():
parser.add_argument("--report", required=True)
parser.add_argument("--timeout", type=int, default=20)
parser.add_argument("--screenshot-on-fail", action="store_true")
parser.add_argument("--use-minicap", action="store_true", help="启用 minicap 高效截图（需已安装辅助工具）")
parser.add_argument("--use-javacap", action="store_true", help="启用 javacap 高效截图（需已安装 Yosemite）")
args = parser.parse_args()
case_path = os.path.abspath(args.case)