UI自动化能力优化

src/main/java/com/example/workutils/biz/mobileuiauto/MobileUiAutomationPlugin.java

@@ -52,6 +52,7 @@ public class MobileUiAutomationPlugin implements Plugin {
                 }
                 latch = new CountDownLatch(1);
                 window = new MobileUiAutomationWindow(automationService, properties, latch);
+                window.onFirstShow();
                 embeddedRoot = window.getContentPane();
                 window.getStage().getScene().setRoot(new Pane());
                 return embeddedRoot;
@@ -66,6 +67,7 @@ public class MobileUiAutomationPlugin implements Plugin {
                 if (embeddedRoot != null && window.getStage().getScene().getRoot() != embeddedRoot) {
                     window.getStage().getScene().setRoot(embeddedRoot);
                 }
+                window.onFirstShow();
                 window.show();
                 new Thread(() -> {
                     try {

src/main/java/com/example/workutils/biz/mobileuiauto/service/MobileAutomationService.java

@@ -9,6 +9,7 @@ import org.springframework.stereotype.Service;
 import javax.annotation.PostConstruct;
 import java.io.*;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -24,9 +25,12 @@ import java.util.concurrent.TimeUnit;
 public class MobileAutomationService {
 
     private static final DateTimeFormatter RUN_ID_FMT = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss");
+    private static final DateTimeFormatter CASE_NAME_FMT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH-mm");
+    private static final DateTimeFormatter REPORT_DIR_FMT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH-mm-ss");
     private static final String[] PYTHON_RESOURCES = {
             "mobile-ui-auto/python/runner.py",
-            "mobile-ui-auto/python/requirements.txt"
+            "mobile-ui-auto/python/requirements.txt",
+            "mobile-ui-auto/python/install_tools.py"
     };
     private static final String[] TEMPLATE_RESOURCES = {
             "mobile-ui-auto/templates/sample-login-flow.yaml"
@@ -41,6 +45,9 @@ public class MobileAutomationService {
     private final Map<String, AutomationRunHandle> running = new ConcurrentHashMap<>();
 
     private RecordingSession recordingSession;
+    
+    // 默认等待时间（毫秒），可通过 UI 配置修改
+    private int defaultWaitTimeMs = 500;
 
     @PostConstruct
     public void init() {
@@ -73,6 +80,67 @@ public class MobileAutomationService {
         return venvDir;
     }
 
+    /**
+     * 设置默认等待时间（毫秒）
+     */
+    public void setDefaultWaitTime(int waitMs) {
+        this.defaultWaitTimeMs = Math.max(100, waitMs); // 最小 100ms
+    }
+
+    /**
+     * 获取当前默认等待时间
+     */
+    public int getDefaultWaitTime() {
+        return defaultWaitTimeMs;
+    }
+
+    /**
+     * 列出执行历史记录（从报告目录读取 result.json）
+     * @param limit 最大返回数量
+     * @return 按时间倒序排列的历史记录
+     */
+    public List<ExecutionHistoryEntry> listExecutionHistory(int limit) {
+        List<ExecutionHistoryEntry> entries = new ArrayList<>();
+        if (!Files.isDirectory(reportsDir)) {
+            return entries;
+        }
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(reportsDir)) {
+            for (Path dir : stream) {
+                if (!Files.isDirectory(dir)) continue;
+                Path resultFile = dir.resolve("result.json");
+                if (!Files.exists(resultFile)) continue;
+                try {
+                    String content = new String(Files.readAllBytes(resultFile), StandardCharsets.UTF_8);
+                    JSONObject json = JSON.parseObject(content);
+                    ExecutionHistoryEntry entry = new ExecutionHistoryEntry();
+                    entry.reportDir = dir;
+                    entry.caseName = json.getString("caseName");
+                    entry.device = json.getString("device");
+                    entry.packageName = json.getString("package");
+                    entry.success = Boolean.TRUE.equals(json.getBoolean("success"));
+                    entry.startTime = json.getString("startTime");
+                    entry.endTime = json.getString("endTime");
+                    entry.steps = json.getJSONArray("steps") != null ? json.getJSONArray("steps").toJavaList(Object.class) : Collections.emptyList();
+                    entries.add(entry);
+                } catch (Exception e) {
+                    log.debug("解析报告失败: {} - {}", resultFile, e.getMessage());
+                }
+            }
+        } catch (IOException e) {
+            log.warn("读取报告目录失败: {}", e.getMessage());
+        }
+        // 按开始时间倒序排列
+        entries.sort((a, b) -> {
+            String ta = a.startTime != null ? a.startTime : "";
+            String tb = b.startTime != null ? b.startTime : "";
+            return tb.compareTo(ta);
+        });
+        if (limit > 0 && entries.size() > limit) {
+            return entries.subList(0, limit);
+        }
+        return entries;
+    }
+
     public void initializeProjectEnvironment(AutomationCallback callback) throws IOException {
         Files.createDirectories(baseDir);
         Files.createDirectories(pythonDir);
@@ -113,6 +181,56 @@ public class MobileAutomationService {
         callback.onEvent("log", "INFO", "Python 依赖安装完成");
     }
 
+    /**
+     * 安装辅助截图工具（minicap/yosemite）到指定设备
+     */
+    public void installAssistTools(String deviceSerial, AutomationCallback callback) throws IOException {
+        Path python = resolvePythonExecutable();
+        if (!Files.exists(python)) {
+            throw new IOException("Python 环境未初始化，请先点击\"初始化环境\"");
+        }
+        String deviceUri = "Android:///" + nullToEmpty(deviceSerial);
+        List<String> command = Arrays.asList(
+                python.toString(),
+                pythonDir.resolve("install_tools.py").toString(),
+                "--device", deviceUri,
+                "--tool", "all",
+                "--marker-dir", baseDir.toString()
+        );
+        callback.onEvent("log", "INFO", "正在安装辅助截图工具到设备: " + deviceSerial);
+        CmdResult result = runStreamingCommand(command, baseDir, 120, callback);
+        if (result.ok) {
+            callback.onEvent("log", "INFO", "辅助工具安装完成");
+        } else {
+            callback.onEvent("log", "WARN", "部分工具安装失败，运行用例时将自动回退到 ADB 截图方式");
+        }
+    }
+
+    /**
+     * 检查指定设备是否已安装 minicap
+     */
+    public boolean isMinicapInstalled(String deviceSerial) {
+        String safeSerial = sanitizeSerial(deviceSerial);
+        Path marker = baseDir.resolve(".minicap-" + safeSerial);
+        return Files.exists(marker);
+    }
+
+    /**
+     * 检查指定设备是否已安装 Yosemite
+     */
+    public boolean isYosemiteInstalled(String deviceSerial) {
+        String safeSerial = sanitizeSerial(deviceSerial);
+        Path marker = baseDir.resolve(".yosemite-" + safeSerial);
+        return Files.exists(marker);
+    }
+
+    private String sanitizeSerial(String serial) {
+        if (serial == null || serial.trim().isEmpty()) {
+            return "default";
+        }
+        return serial.trim().replace(":", "_").replace("/", "_");
+    }
+
     public EnvironmentCheckResult checkEnvironment() {
         EnvironmentCheckResult result = new EnvironmentCheckResult();
         Path python = resolvePythonExecutable();
@@ -184,8 +302,11 @@ public class MobileAutomationService {
 
     public AutomationRunHandle runCase(AutomationRunConfig config, AutomationCallback callback) throws IOException {
         validateRunConfig(config);
-        String runId = LocalDateTime.now().format(RUN_ID_FMT);
-        Path reportDir = reportsDir.resolve(runId);
+        LocalDateTime now = LocalDateTime.now();
+        String runId = now.format(RUN_ID_FMT);
+        // 报告目录使用可读格式: 报告 2026-07-06 19-13-02
+        String reportDirName = "报告 " + now.format(REPORT_DIR_FMT);
+        Path reportDir = reportsDir.resolve(reportDirName);
         Files.createDirectories(reportDir);
 
         List<String> command = new ArrayList<>();
@@ -206,9 +327,18 @@ public class MobileAutomationService {
         if (config.isScreenshotOnFail()) {
             command.add("--screenshot-on-fail");
         }
+        // 如果设备已安装辅助工具，启用高效截图模式
+        if (isMinicapInstalled(config.getDeviceSerial())) {
+            command.add("--use-minicap");
+            callback.onEvent("log", "INFO", "minicap 已安装，启用高效截图模式");
+        } else if (isYosemiteInstalled(config.getDeviceSerial())) {
+            command.add("--use-javacap");
+            callback.onEvent("log", "INFO", "Yosemite 已安装，启用 javacap 高效截图模式");
+        }
 
         ProcessBuilder pb = new ProcessBuilder(command);
         pb.directory(baseDir.toFile());
+        pb.environment().put("PYTHONIOENCODING", "utf-8");
         pb.redirectErrorStream(false);
         Process process = pb.start();
         AutomationRunHandle handle = new AutomationRunHandle(runId, process);
@@ -253,10 +383,86 @@ public class MobileAutomationService {
     }
 
     public synchronized void startRecording(String deviceSerial, String packageName) throws IOException {
-        String runId = "record-" + LocalDateTime.now().format(RUN_ID_FMT);
-        Path recordDir = reportsDir.resolve(runId);
+        LocalDateTime now = LocalDateTime.now();
+        String runId = "record-" + now.format(RUN_ID_FMT);
+        // 报告目录使用可读格式: 报告 2026-07-06 19-13-02
+        String reportDirName = "报告 " + now.format(REPORT_DIR_FMT);
+        Path recordDir = reportsDir.resolve(reportDirName);
         Files.createDirectories(recordDir.resolve("screenshots"));
-        recordingSession = new RecordingSession(runId, deviceSerial, packageName, recordDir);
+        
+        // 如果未提供包名，尝试从设备获取当前前台应用
+        String actualPackageName = packageName;
+        if (actualPackageName == null || actualPackageName.trim().isEmpty()) {
+            actualPackageName = getCurrentForegroundApp(deviceSerial);
+            if (actualPackageName != null) {
+                log.info("自动检测到前台应用: {}", actualPackageName);
+            }
+        }
+        
+        recordingSession = new RecordingSession(runId, deviceSerial, actualPackageName, recordDir, now);
+    }
+
+    /**
+     * 获取设备当前前台应用的包名（排除桌面启动器）
+     */
+    public String getCurrentForegroundApp(String deviceSerial) {
+        try {
+            List<String> command = new ArrayList<>();
+            command.add(resolveAdbExecutable().orElse(Path.of("adb")).toString());
+            if (deviceSerial != null && !deviceSerial.trim().isEmpty()) {
+                command.add("-s");
+                command.add(deviceSerial.trim());
+            }
+            // 使用 dumpsys window 获取当前焦点窗口
+            command.addAll(Arrays.asList("shell", "dumpsys", "window", "|", "grep", "mCurrentFocus"));
+            
+            CmdResult result = runCommand(command, null, 5);
+            if (result.ok && result.output != null && !result.output.isEmpty()) {
+                // 输出格式类似: mCurrentFocus=Window{... u0 com.example.app/com.example.app.MainActivity}
+                String output = result.output.trim();
+                java.util.regex.Pattern pattern = java.util.regex.Pattern.compile("u\\d+\\s+([a-zA-Z0-9._]+)");
+                java.util.regex.Matcher matcher = pattern.matcher(output);
+                if (matcher.find()) {
+                    String packageName = matcher.group(1);
+                    // 排除桌面启动器包名
+                    if (isLauncherPackage(packageName)) {
+                        log.info("检测到桌面启动器包名: {}，跳过自动填充", packageName);
+                        return null;
+                    }
+                    return packageName;
+                }
+            }
+        } catch (Exception e) {
+            log.warn("获取前台应用失败: {}", e.getMessage());
+        }
+        return null;
+    }
+
+    /**
+     * 判断是否为桌面启动器包名
+     */
+    private static final Set<String> LAUNCHER_PACKAGES = Set.of(
+            "com.miui.home",                    // 小米
+            "com.android.launcher",             // AOSP
+            "com.android.launcher3",            // AOSP
+            "com.sec.android.app.launcher",     // 三星
+            "com.huawei.android.launcher",      // 华为
+            "com.oppo.launcher",                // OPPO
+            "com.coloros.launcher",             // OPPO ColorOS
+            "com.bbk.launcher",                 // Vivo
+            "com.vivo.launcher",                // Vivo
+            "com.oneplus.launcher",             // 一加
+            "com.google.android.apps.nexuslauncher", // Pixel
+            "com.teslacoilsw.launcher",         // Nova Launcher
+            "com.motorola.launcher3"            // 摩托罗拉
+    );
+
+    private boolean isLauncherPackage(String packageName) {
+        if (packageName == null) return false;
+        String lower = packageName.toLowerCase(Locale.ROOT);
+        if (LAUNCHER_PACKAGES.contains(lower)) return true;
+        // 通配匹配：包含 launcher 关键字的包名
+        return lower.contains(".launcher") || lower.endsWith(".home");
     }
 
     public synchronized boolean isRecording() {
@@ -267,8 +473,13 @@ public class MobileAutomationService {
         if (recordingSession == null) {
             throw new IOException("尚未开始录制");
         }
-        Path screenshot = recordingSession.recordDir.resolve("screenshots/current.png");
+        // 使用带时间戳的文件名避免覆盖，同时保留 current.png 作为最新截图的快捷方式
+        String timestamp = LocalDateTime.now().format(RUN_ID_FMT);
+        Path screenshot = recordingSession.recordDir.resolve("screenshots/screenshot-" + timestamp + ".png");
         captureScreenshot(recordingSession.deviceSerial, screenshot);
+        // 同步创建 current.png 软链接（或复制），供实时显示使用
+        Path current = recordingSession.recordDir.resolve("screenshots/current.png");
+        Files.copy(screenshot, current, StandardCopyOption.REPLACE_EXISTING);
         return screenshot;
     }
 
@@ -279,7 +490,7 @@ public class MobileAutomationService {
         int x = scale(imageX, imageWidth, screenWidth);
         int y = scale(imageY, imageHeight, screenHeight);
         runAdb(recordingSession.deviceSerial, Arrays.asList("shell", "input", "tap", String.valueOf(x), String.valueOf(y)), 10);
-        recordingSession.addTap(x, y, screenWidth, screenHeight);
+        recordingSession.addTap(x, y, screenWidth, screenHeight, defaultWaitTimeMs);
     }
 
     public synchronized void recordInput(String text) throws IOException {
@@ -290,7 +501,7 @@ public class MobileAutomationService {
             text = "";
         }
         runAdb(recordingSession.deviceSerial, Arrays.asList("shell", "input", "text", escapeAdbInput(text)), 10);
-        recordingSession.addInput(text);
+        recordingSession.addInput(text, defaultWaitTimeMs);
     }
 
     public synchronized void recordBack() throws IOException {
@@ -298,7 +509,7 @@ public class MobileAutomationService {
             throw new IOException("尚未开始录制");
         }
         runAdb(recordingSession.deviceSerial, Arrays.asList("shell", "input", "keyevent", "BACK"), 10);
-        recordingSession.addKeyevent("BACK");
+        recordingSession.addKeyevent("BACK", defaultWaitTimeMs);
     }
 
     public synchronized void recordSwipe(int startX, int startY, int endX, int endY, int durationMs) throws IOException {
@@ -307,22 +518,362 @@ public class MobileAutomationService {
         }
         runAdb(recordingSession.deviceSerial, Arrays.asList("shell", "input", "swipe",
                 String.valueOf(startX), String.valueOf(startY), String.valueOf(endX), String.valueOf(endY), String.valueOf(durationMs)), 10);
-        recordingSession.addSwipe(startX, startY, endX, endY, durationMs);
+        recordingSession.addSwipe(startX, startY, endX, endY, durationMs, defaultWaitTimeMs);
+    }
+
+    /**
+     * 撤销上一步录制操作
+     * @return 被撤销的步骤描述，null 表示没有可撤销的步骤
+     */
+    public synchronized String undoLastStep() {
+        if (recordingSession == null || recordingSession.steps.isEmpty()) {
+            return null;
+        }
+        JSONObject removed = recordingSession.steps.remove(recordingSession.steps.size() - 1);
+        recordingSession.index--;
+        // 返回步骤名称用于显示
+        return removed.getString("name");
+    }
+
+    /**
+     * 更新指定步骤的等待时间
+     * @param index 步骤索引（从0开始）
+     * @param waitMs 新的等待时间（毫秒）
+     * @return 是否成功
+     */
+    public synchronized boolean updateStepWaitTime(int index, int waitMs) {
+        if (recordingSession == null || index < 0 || index >= recordingSession.steps.size()) {
+            return false;
+        }
+        JSONObject step = recordingSession.steps.get(index);
+        step.put("waitAfterMs", waitMs);
+        return true;
+    }
+
+    /**
+     * 删除指定索引的步骤
+     * @param index 步骤索引（从0开始）
+     * @return 是否成功
+     */
+    public synchronized boolean deleteStep(int index) {
+        if (recordingSession == null || index < 0 || index >= recordingSession.steps.size()) {
+            return false;
+        }
+        recordingSession.steps.remove(index);
+        // 重新编号后续步骤
+        for (int i = index; i < recordingSession.steps.size(); i++) {
+            JSONObject step = recordingSession.steps.get(i);
+            String name = step.getString("name");
+            if (name != null) {
+                // 将 tap-N、input-N 等改为正确的序号
+                String newName = name.replaceAll("-(\\d+)$", "-" + (i + 1));
+                step.put("name", newName);
+            }
+        }
+        recordingSession.index = recordingSession.steps.size() + 1;
+        return true;
+    }
+
+    /**
+     * 设置指定步骤的页面特征
+     * @param index 步骤索引（从0开始）
+     * @param featureName 页面特征名称（如"login-page"、"ad-popup"等）
+     * @param screenshotPath 截图路径（可选，null表示不设置截图）
+     * @return 是否成功
+     */
+    public synchronized boolean setStepPageFeature(int index, String featureName, String screenshotPath) {
+        if (recordingSession == null || index < 0 || index >= recordingSession.steps.size()) {
+            return false;
+        }
+        JSONObject step = recordingSession.steps.get(index);
+        if (featureName == null || featureName.trim().isEmpty()) {
+            step.remove("pageFeature");
+            step.remove("pageFeatureScreenshot");
+        } else {
+            step.put("pageFeature", featureName.trim());
+            if (screenshotPath != null && !screenshotPath.isEmpty()) {
+                step.put("pageFeatureScreenshot", screenshotPath);
+            }
+        }
+        return true;
+    }
+
+    /**
+     * 获取指定步骤的页面特征信息
+     * @param index 步骤索引（从0开始）
+     * @return 页面特征名称，如果没有则返回 null
+     */
+    public synchronized String getStepPageFeature(int index) {
+        if (recordingSession == null || index < 0 || index >= recordingSession.steps.size()) {
+            return null;
+        }
+        JSONObject step = recordingSession.steps.get(index);
+        return step.getString("pageFeature");
+    }
+
+    /**
+     * 获取指定步骤的完整JSON信息
+     * @param index 步骤索引（从0开始）
+     * @return 步骤的 JSONObject，如果无效则返回 null
+     */
+    public synchronized JSONObject getStepInfo(int index) {
+        if (recordingSession == null || index < 0 || index >= recordingSession.steps.size()) {
+            return null;
+        }
+        return recordingSession.steps.get(index);
+    }
+
+    /**
+     * 获取设备当前页面的 UI 层级信息
+     * @param deviceSerial 设备序列号
+     * @return UI 元素列表，每个元素包含 resourceId, text, contentDesc, className, bounds
+     */
+    public List<JSONObject> dumpUiHierarchy(String deviceSerial) throws IOException {
+        // 1. 在设备上生成 UI dump
+        Path localDump = baseDir.resolve("window_dump.xml");
+        runAdb(deviceSerial, Arrays.asList("shell", "uiautomator", "dump", "/sdcard/window_dump.xml"), 15);
+        // 2. 拉取到本地
+        runAdb(deviceSerial, Arrays.asList("pull", "/sdcard/window_dump.xml", localDump.toString()), 10);
+        // 3. 解析 XML
+        List<JSONObject> elements = new ArrayList<>();
+        if (!Files.exists(localDump)) {
+            throw new IOException("UI dump 文件不存在");
+        }
+        try {
+            String xmlContent = new String(Files.readAllBytes(localDump), StandardCharsets.UTF_8);
+            // 简单解析 XML 中的 node 元素
+            javax.xml.parsers.DocumentBuilderFactory factory = javax.xml.parsers.DocumentBuilderFactory.newInstance();
+            javax.xml.parsers.DocumentBuilder builder = factory.newDocumentBuilder();
+            org.w3c.dom.Document doc = builder.parse(new ByteArrayInputStream(xmlContent.getBytes(StandardCharsets.UTF_8)));
+            org.w3c.dom.NodeList nodes = doc.getElementsByTagName("node");
+            for (int i = 0; i < nodes.getLength(); i++) {
+                org.w3c.dom.Element node = (org.w3c.dom.Element) nodes.item(i);
+                JSONObject elem = new JSONObject();
+                elem.put("resourceId", node.getAttribute("resource-id"));
+                elem.put("text", node.getAttribute("text"));
+                elem.put("contentDesc", node.getAttribute("content-desc"));
+                elem.put("className", node.getAttribute("class"));
+                elem.put("bounds", node.getAttribute("bounds"));
+                elem.put("clickable", "true".equals(node.getAttribute("clickable")));
+                elem.put("enabled", "true".equals(node.getAttribute("enabled")));
+                // 解析 bounds: [left,top][right,bottom]
+                String bounds = node.getAttribute("bounds");
+                if (bounds != null && !bounds.isEmpty()) {
+                    java.util.regex.Matcher m = java.util.regex.Pattern.compile("\\[(\\d+),(\\d+)\\]\\[(\\d+),(\\d+)\\]").matcher(bounds);
+                    if (m.find()) {
+                        elem.put("left", Integer.parseInt(m.group(1)));
+                        elem.put("top", Integer.parseInt(m.group(2)));
+                        elem.put("right", Integer.parseInt(m.group(3)));
+                        elem.put("bottom", Integer.parseInt(m.group(4)));
+                    }
+                }
+                elements.add(elem);
+            }
+        } catch (javax.xml.parsers.ParserConfigurationException | org.xml.sax.SAXException e) {
+            throw new IOException("解析 UI dump XML 失败: " + e.getMessage());
+        } finally {
+            try { Files.deleteIfExists(localDump); } catch (IOException ignored) {}
+        }
+        return elements;
+    }
+
+    /**
+     * 识别指定坐标处的 UI 元素
+     * @param deviceSerial 设备序列号
+     * @param x 屏幕 X 坐标
+     * @param y 屏幕 Y 坐标
+     * @return 匹配的元素列表（按优先级排序：resource-id > content-desc > text）
+     */
+    public List<JSONObject> identifyElement(String deviceSerial, int x, int y) throws IOException {
+        List<JSONObject> allElements = dumpUiHierarchy(deviceSerial);
+        List<JSONObject> matched = new ArrayList<>();
+        for (JSONObject elem : allElements) {
+            Integer left = elem.getInteger("left");
+            Integer top = elem.getInteger("top");
+            Integer right = elem.getInteger("right");
+            Integer bottom = elem.getInteger("bottom");
+            if (left != null && top != null && right != null && bottom != null) {
+                if (x >= left && x <= right && y >= top && y <= bottom) {
+                    // 计算面积用于排序（优先选择更精确的小元素）
+                    int area = (right - left) * (bottom - top);
+                    elem.put("area", area);
+                    matched.add(elem);
+                }
+            }
+        }
+        // 按面积升序排序（面积越小越精确）
+        matched.sort((a, b) -> a.getIntValue("area") - b.getIntValue("area"));
+        return matched;
+    }
+
+    /**
+     * 更新指定步骤的目标元素信息（元素优先+坐标兑底）
+     * @param index 步骤索引
+     * @param element 识别到的元素信息（来自 identifyElement）
+     * @return 是否成功
+     */
+    public synchronized boolean updateStepElement(int index, JSONObject element) {
+        if (recordingSession == null || index < 0 || index >= recordingSession.steps.size()) {
+            return false;
+        }
+        JSONObject step = recordingSession.steps.get(index);
+        JSONObject target = step.getJSONObject("target");
+        if (target == null) {
+            target = new JSONObject();
+            step.put("target", target);
+        }
+        // 设置 Poco 查询信息
+        JSONObject poco = new JSONObject();
+        String resourceId = element.getString("resourceId");
+        String contentDesc = element.getString("contentDesc");
+        String text = element.getString("text");
+        if (resourceId != null && !resourceId.isEmpty()) {
+            poco.put("name", resourceId);
+        } else if (contentDesc != null && !contentDesc.isEmpty()) {
+            poco.put("name", contentDesc);
+        } else if (text != null && !text.isEmpty()) {
+            poco.put("text", text);
+        } else {
+            return false; // 无可识别属性
+        }
+        target.put("preferred", "poco");
+        target.put("poco", poco);
+        return true;
+    }
+
+    /**
+     * 更新已保存用例文件中的步骤元素信息
+     * @param caseFile 用例文件路径
+     * @param stepIndex 步骤索引
+     * @param element 识别到的元素信息
+     * @return 是否成功
+     */
+    public boolean updateCaseStepElement(Path caseFile, int stepIndex, JSONObject element) throws IOException {
+        if (caseFile == null || !Files.exists(caseFile)) {
+            return false;
+        }
+        String content = new String(Files.readAllBytes(caseFile), StandardCharsets.UTF_8);
+        JSONObject caseJson = JSON.parseObject(content);
+        com.alibaba.fastjson.JSONArray steps = caseJson.getJSONArray("steps");
+        if (steps == null || stepIndex < 0 || stepIndex >= steps.size()) {
+            return false;
+        }
+        JSONObject step = steps.getJSONObject(stepIndex);
+        JSONObject target = step.getJSONObject("target");
+        if (target == null) {
+            target = new JSONObject();
+            step.put("target", target);
+        }
+        JSONObject poco = new JSONObject();
+        String resourceId = element.getString("resourceId");
+        String contentDesc = element.getString("contentDesc");
+        String text = element.getString("text");
+        if (resourceId != null && !resourceId.isEmpty()) {
+            poco.put("name", resourceId);
+        } else if (contentDesc != null && !contentDesc.isEmpty()) {
+            poco.put("name", contentDesc);
+        } else if (text != null && !text.isEmpty()) {
+            poco.put("text", text);
+        } else {
+            return false;
+        }
+        target.put("preferred", "poco");
+        target.put("poco", poco);
+        Files.write(caseFile, caseJson.toJSONString().getBytes(StandardCharsets.UTF_8));
+        return true;
+    }
+
+    /**
+     * 对已保存的用例文件进行 UI dump 并识别指定步骤的元素
+     * @param deviceSerial 设备序列号
+     * @param caseFile 用例文件
+     * @param stepIndex 步骤索引
+     * @return 匹配的元素列表
+     */
+    public List<JSONObject> identifyCaseStepElement(String deviceSerial, Path caseFile, int stepIndex) throws IOException {
+        if (caseFile == null || !Files.exists(caseFile)) {
+            throw new IOException("用例文件不存在");
+        }
+        String content = new String(Files.readAllBytes(caseFile), StandardCharsets.UTF_8);
+        JSONObject caseJson = JSON.parseObject(content);
+        com.alibaba.fastjson.JSONArray steps = caseJson.getJSONArray("steps");
+        if (steps == null || stepIndex < 0 || stepIndex >= steps.size()) {
+            throw new IOException("步骤索引无效");
+        }
+        JSONObject step = steps.getJSONObject(stepIndex);
+        JSONObject target = step.getJSONObject("target");
+        if (target == null || !"tap".equals(step.getString("action"))) {
+            return Collections.emptyList();
+        }
+        int x = target.getIntValue("x");
+        int y = target.getIntValue("y");
+        return identifyElement(deviceSerial, x, y);
     }
 
     public synchronized Path stopRecording() throws IOException {
         if (recordingSession == null) {
             throw new IOException("尚未开始录制");
         }
-        Path file = casesDir.resolve(recordingSession.runId + ".yaml");
-        Files.write(file, recordingSession.toYaml().getBytes(StandardCharsets.UTF_8));
-        Path latest = casesDir.resolve("latest-recorded-flow.yaml");
+        // 生成可读用例文件名: 2026-07-06 19-13 用例（N步）.json
+        int stepCount = recordingSession.steps.size();
+        String caseName = recordingSession.startTime.format(CASE_NAME_FMT) + " 用例（" + stepCount + "步）";
+        Path file = casesDir.resolve(caseName + ".json");
+        // 更新 JSON 内的 name 字段
+        recordingSession.caseName = caseName;
+        Files.write(file, recordingSession.toJSON().getBytes(StandardCharsets.UTF_8));
+        Path latest = casesDir.resolve("latest-recorded-flow.json");
         Files.copy(file, latest, StandardCopyOption.REPLACE_EXISTING);
         recordingSession = null;
         return file;
     }
 
-    private void captureScreenshot(String serial, Path output) throws IOException {
+    /**
+     * 重命名用例文件
+     */
+    public Path renameCase(Path oldFile, String newName) throws IOException {
+        if (oldFile == null || !Files.exists(oldFile)) {
+            throw new IOException("文件不存在");
+        }
+        // 清理文件名中的非法字符
+        String safeName = newName.replaceAll("[\\\\/:*?\"<>|]", "-");
+        Path newFile = oldFile.getParent().resolve(safeName + ".json");
+        if (Files.exists(newFile) && !newFile.equals(oldFile)) {
+            throw new IOException("文件名已存在: " + safeName);
+        }
+        // 更新 JSON 内部的 name 字段
+        try {
+            String content = new String(Files.readAllBytes(oldFile), StandardCharsets.UTF_8);
+            JSONObject json = JSON.parseObject(content);
+            json.put("name", safeName);
+            Files.write(oldFile, json.toJSONString().getBytes(StandardCharsets.UTF_8));
+        } catch (Exception e) {
+            log.warn("更新用例name字段失败: {}", e.getMessage());
+        }
+        if (!newFile.equals(oldFile)) {
+            Files.move(oldFile, newFile, StandardCopyOption.REPLACE_EXISTING);
+        }
+        return newFile;
+    }
+
+    /**
+     * 重命名报告目录
+     */
+    public Path renameReport(Path oldDir, String newName) throws IOException {
+        if (oldDir == null || !Files.isDirectory(oldDir)) {
+            throw new IOException("报告目录不存在");
+        }
+        String safeName = newName.replaceAll("[\\\\/:*?\"<>|]", "-");
+        Path newDir = oldDir.getParent().resolve(safeName);
+        if (Files.exists(newDir) && !newDir.equals(oldDir)) {
+            throw new IOException("目录名已存在: " + safeName);
+        }
+        if (!newDir.equals(oldDir)) {
+            Files.move(oldDir, newDir, StandardCopyOption.REPLACE_EXISTING);
+        }
+        return newDir;
+    }
+
+    public void captureScreenshot(String serial, Path output) throws IOException {
         List<String> command = new ArrayList<>();
         command.add(resolveAdbExecutable().orElse(Path.of("adb")).toString());
         if (serial != null && !serial.trim().isEmpty()) {
@@ -364,7 +915,7 @@ public class MobileAutomationService {
 
     private boolean isCaseFile(Path path) {
         String name = path.getFileName().toString().toLowerCase(Locale.ROOT);
-        return name.endsWith(".yaml") || name.endsWith(".yml") || name.endsWith(".py") || name.endsWith(".air");
+        return name.endsWith(".json") || name.endsWith(".yaml") || name.endsWith(".yml") || name.endsWith(".py") || name.endsWith(".air");
     }
 
     private void addCheck(EnvironmentCheckResult result, String name, List<String> command) {
@@ -388,6 +939,19 @@ public class MobileAutomationService {
         try (BufferedReader reader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8))) {
             String line;
             while ((line = reader.readLine()) != null) {
+                // 过滤 Airtest 内部 DEBUG 日志（Airtest 将 DEBUG 日志写到 stderr）
+                if (line.contains("[DEBUG]") || line.contains("[DEBUG]<")) {
+                    continue; // 跳过 DEBUG 级别日志
+                }
+                // 过滤 Airtest 内部的 INFO 日志（也写到 stderr）
+                if (line.matches(".*\\[INFO\\]<airtest\\..*")) {
+                    continue; // 跳过 Airtest 内部 INFO 日志
+                }
+                // 过滤 Python warnings
+                if (line.contains("UserWarning") || line.contains("warnings.warn")) {
+                    continue;
+                }
+                // 只有真正的错误才显示
                 callback.onEvent("log", "ERROR", line);
             }
         } catch (IOException e) {
@@ -413,6 +977,20 @@ public class MobileAutomationService {
         }
     }
 
+    /**
+     * 检查指定设备是否仍然在线
+     */
+    public boolean isDeviceConnected(String serial) {
+        try {
+            Optional<Path> adb = resolveAdbExecutable();
+            if (adb.isEmpty()) return false;
+            CmdResult result = runCommand(Arrays.asList(adb.get().toString(), "-s", serial, "get-state"), null, 5);
+            return result.ok && result.output.trim().equals("device");
+        } catch (Exception e) {
+            return false;
+        }
+    }
+
     private void runAdb(String serial, List<String> args, int timeoutSeconds) throws IOException {
         CmdResult result = runCommand(buildAdbCommand(serial, args), null, timeoutSeconds);
         if (!result.ok) {
@@ -476,6 +1054,8 @@ public class MobileAutomationService {
             if (workingDir != null) {
                 pb.directory(workingDir.toFile());
             }
+            // 确保 Python 输出使用 UTF-8 编码，避免中文乱码
+            pb.environment().put("PYTHONIOENCODING", "utf-8");
             pb.redirectErrorStream(true);
             process = pb.start();
             Process proc = process;
@@ -679,73 +1259,113 @@ public class MobileAutomationService {
         final String deviceSerial;
         final String packageName;
         final Path recordDir;
-        final List<String> steps = new ArrayList<>();
+        final LocalDateTime startTime;
+        String caseName; // 停止录制时设置
+        final List<JSONObject> steps = new ArrayList<>();
         int index = 1;
 
-        RecordingSession(String runId, String deviceSerial, String packageName, Path recordDir) {
+        RecordingSession(String runId, String deviceSerial, String packageName, Path recordDir, LocalDateTime startTime) {
             this.runId = runId;
             this.deviceSerial = deviceSerial;
             this.packageName = packageName;
             this.recordDir = recordDir;
+            this.startTime = startTime;
+            this.caseName = runId; // 默认值
         }
 
-        void addTap(int x, int y, int screenWidth, int screenHeight) {
-            steps.add("  - name: tap-" + index + "\n"
-                    + "    action: tap\n"
-                    + "    target:\n"
-                    + "      type: coord\n"
-                    + "      preferred: coord\n"
-                    + "      x: " + x + "\n"
-                    + "      y: " + y + "\n"
-                    + "      screenWidth: " + screenWidth + "\n"
-                    + "      screenHeight: " + screenHeight + "\n"
-                    + "    waitAfterMs: 500");
+        void addTap(int x, int y, int screenWidth, int screenHeight, int waitAfterMs) {
+            JSONObject step = new JSONObject();
+            step.put("name", "tap-" + index);
+            step.put("action", "tap");
+            
+            JSONObject target = new JSONObject();
+            target.put("type", "coord");
+            target.put("preferred", "coord");
+            target.put("x", x);
+            target.put("y", y);
+            target.put("screenWidth", screenWidth);
+            target.put("screenHeight", screenHeight);
+            step.put("target", target);
+            step.put("waitAfterMs", waitAfterMs);
+            // 可选：页面特征（用于执行时验证当前页面）
+            // step.put("pageFeature", "login-page");
+            
+            steps.add(step);
             index++;
         }
 
-        void addInput(String text) {
-            steps.add("  - name: input-" + index + "\n"
-                    + "    action: input_text\n"
-                    + "    text: \"" + yamlEscape(text) + "\"\n"
-                    + "    waitAfterMs: 500");
+        void addInput(String text, int waitAfterMs) {
+            JSONObject step = new JSONObject();
+            step.put("name", "input-" + index);
+            step.put("action", "input_text");
+            step.put("text", text);
+            step.put("waitAfterMs", waitAfterMs);
+            
+            steps.add(step);
             index++;
         }
 
-        void addKeyevent(String key) {
-            steps.add("  - name: key-" + index + "\n"
-                    + "    action: keyevent\n"
-                    + "    key: " + key + "\n"
-                    + "    waitAfterMs: 500");
+        void addKeyevent(String key, int waitAfterMs) {
+            JSONObject step = new JSONObject();
+            step.put("name", "key-" + index);
+            step.put("action", "keyevent");
+            step.put("key", key);
+            step.put("waitAfterMs", waitAfterMs);
+            
+            steps.add(step);
             index++;
         }
 
-        void addSwipe(int startX, int startY, int endX, int endY, int durationMs) {
-            steps.add("  - name: swipe-" + index + "\n"
-                    + "    action: swipe\n"
-                    + "    start: {x: " + startX + ", y: " + startY + "}\n"
-                    + "    end: {x: " + endX + ", y: " + endY + "}\n"
-                    + "    durationMs: " + durationMs + "\n"
-                    + "    waitAfterMs: 500");
+        void addSwipe(int startX, int startY, int endX, int endY, int durationMs, int waitAfterMs) {
+            JSONObject step = new JSONObject();
+            step.put("name", "swipe-" + index);
+            step.put("action", "swipe");
+            
+            JSONObject start = new JSONObject();
+            start.put("x", startX);
+            start.put("y", startY);
+            step.put("start", start);
+            
+            JSONObject end = new JSONObject();
+            end.put("x", endX);
+            end.put("y", endY);
+            step.put("end", end);
+            
+            step.put("durationMs", durationMs);
+            step.put("waitAfterMs", waitAfterMs);
+            
+            steps.add(step);
             index++;
         }
 
-        String toYaml() {
-            StringBuilder yaml = new StringBuilder();
-            yaml.append("name: ").append(runId).append("\n");
-            yaml.append("app:\n");
-            yaml.append("  package: ").append(packageName == null ? "" : packageName).append("\n");
-            yaml.append("  activity: \n");
-            yaml.append("device:\n");
-            yaml.append("  connect: Android:///");
-            yaml.append(deviceSerial == null ? "" : deviceSerial).append("\n");
-            yaml.append("settings:\n");
-            yaml.append("  timeout: 20\n");
-            yaml.append("  screenshotOnFail: true\n");
-            yaml.append("steps:\n");
-            for (String step : steps) {
-                yaml.append(step).append("\n");
-            }
-            return yaml.toString();
+        /**
+         * 生成 JSON 格式的用例文件
+         */
+        String toJSON() {
+            JSONObject caseJson = new JSONObject();
+            caseJson.put("name", caseName);
+            
+            // App 信息
+            JSONObject app = new JSONObject();
+            app.put("package", packageName == null ? "" : packageName);
+            app.put("activity", "");
+            caseJson.put("app", app);
+            
+            // 设备信息
+            JSONObject device = new JSONObject();
+            device.put("connect", "Android:///" + (deviceSerial == null ? "" : deviceSerial));
+            caseJson.put("device", device);
+            
+            // 设置
+            JSONObject settings = new JSONObject();
+            settings.put("timeout", 20);
+            settings.put("screenshotOnFail", true);
+            caseJson.put("settings", settings);
+            
+            // 步骤列表
+            caseJson.put("steps", steps);
+            
+            return caseJson.toJSONString();
         }
 
         private String yamlEscape(String text) {

src/main/java/com/example/workutils/biz/mobileuiauto/service/model/ExecutionHistoryEntry.java

@@ -0,0 +1,36 @@
+package com.example.workutils.biz.mobileuiauto.service.model;
+
+import java.nio.file.Path;
+import java.util.List;
+
+/**
+ * 执行历史记录条目
+ */
+public class ExecutionHistoryEntry {
+    /** 报告目录 */
+    public Path reportDir;
+    /** 用例名称 */
+    public String caseName;
+    /** 设备标识 */
+    public String device;
+    /** 包名 */
+    public String packageName;
+    /** 是否成功 */
+    public boolean success;
+    /** 开始时间（ISO 格式字符串） */
+    public String startTime;
+    /** 结束时间（ISO 格式字符串） */
+    public String endTime;
+    /** 步骤列表 */
+    public List<Object> steps;
+
+    @Override
+    public String toString() {
+        String icon = success ? "✅" : "❌";
+        String time = startTime != null && startTime.length() >= 16 
+            ? startTime.substring(0, 16).replace("T", " ") 
+            : startTime;
+        int stepCount = steps != null ? steps.size() : 0;
+        return icon + " " + caseName + "  (" + stepCount + "步)  " + time;
+    }
+}