From 39f2a7e17e23ec6238918516e50ede691cb6f251 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AE=B6=E8=AE=AD?= Date: Wed, 8 Jul 2026 18:04:48 +0800 Subject: [PATCH] =?UTF-8?q?UI=E8=87=AA=E5=8A=A8=E5=8C=96=E8=83=BD=E5=8A=9B?= =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../MobileUiAutomationPlugin.java | 2 + .../MobileUiAutomationWindow.java | 1297 ++++++++++++++++- .../service/MobileAutomationService.java | 746 +++++++++- .../service/model/ExecutionHistoryEntry.java | 36 + src/main/resources/css/light-theme.css | 15 + .../mobile-ui-auto/python/install_tools.py | 286 ++++ .../resources/mobile-ui-auto/python/runner.py | 127 +- 7 files changed, 2361 insertions(+), 148 deletions(-) create mode 100644 src/main/java/com/example/workutils/biz/mobileuiauto/service/model/ExecutionHistoryEntry.java create mode 100644 src/main/resources/mobile-ui-auto/python/install_tools.py diff --git a/src/main/java/com/example/workutils/biz/mobileuiauto/MobileUiAutomationPlugin.java b/src/main/java/com/example/workutils/biz/mobileuiauto/MobileUiAutomationPlugin.java index 5c3a48b..39bed4b 100644 --- a/src/main/java/com/example/workutils/biz/mobileuiauto/MobileUiAutomationPlugin.java +++ b/src/main/java/com/example/workutils/biz/mobileuiauto/MobileUiAutomationPlugin.java @@ -52,6 +52,7 @@ public class MobileUiAutomationPlugin implements Plugin { } latch = new CountDownLatch(1); window = new MobileUiAutomationWindow(automationService, properties, latch); + window.onFirstShow(); embeddedRoot = window.getContentPane(); window.getStage().getScene().setRoot(new Pane()); return embeddedRoot; @@ -66,6 +67,7 @@ public class MobileUiAutomationPlugin implements Plugin { if (embeddedRoot != null && window.getStage().getScene().getRoot() != embeddedRoot) { window.getStage().getScene().setRoot(embeddedRoot); } + window.onFirstShow(); window.show(); new Thread(() -> { try { diff --git a/src/main/java/com/example/workutils/biz/mobileuiauto/MobileUiAutomationWindow.java b/src/main/java/com/example/workutils/biz/mobileuiauto/MobileUiAutomationWindow.java index b4bf48d..0f23e32 100644 --- a/src/main/java/com/example/workutils/biz/mobileuiauto/MobileUiAutomationWindow.java +++ b/src/main/java/com/example/workutils/biz/mobileuiauto/MobileUiAutomationWindow.java @@ -6,6 +6,7 @@ import com.example.workutils.system.config.WorkUtilsProperties; import com.example.workutils.system.support.SystemOpenHelper; import com.example.workutils.system.ui.StyledButton; import com.example.workutils.system.ui.Theme; +import com.alibaba.fastjson.JSONObject; import javafx.application.Platform; import javafx.collections.FXCollections; import javafx.geometry.Insets; @@ -26,9 +27,14 @@ import javafx.stage.Stage; import java.awt.Desktop; import java.io.File; import java.net.URI; +import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.concurrent.CountDownLatch; +import java.util.prefs.Preferences; +import java.util.stream.Collectors; public class MobileUiAutomationWindow { @@ -38,20 +44,40 @@ public class MobileUiAutomationWindow { private final Stage stage = new Stage(); private ComboBox deviceCombo; - private TextField packageField; - private TextField activityField; private ListView caseList; + private ListView historyList; private TextArea logArea; private Label statusLabel; private Button downloadEnvButton; private Button initEnvButton; - private Button runButton; - private Button stopButton; + private Button installToolsButton; + private Label toolsStatusLabel; + private MenuItem installToolsItem; + private Button runStopButton; // 运行/停止 合并按钮 private AutomationRunHandle currentRun; + private boolean isRunning = false; private ImageView phoneImageView; private Label recordStatusLabel; private TextField recordInputField; + private TextField defaultWaitField; + private TextField recordPackageField; + private TextField recordActivityField; + private Button recordToggleButton; // 开始录制/停止生成 合并按钮 + private boolean isRecording = false; + + // ==================== 录制步骤列表 ==================== + private ListView recordStepList; + private final java.util.List recordedSteps = new java.util.ArrayList<>(); + + // ==================== 实时截图轮询 ==================== + private volatile Thread screenshotThread; + private volatile boolean screenshotRunning = false; + + // ==================== 设备状态 ==================== + private Label deviceStatusLabel; + private static final String PREF_LAST_DEVICE = "mobile.lastDeviceSerial"; + private final Preferences preferences = Preferences.userNodeForPackage(MobileUiAutomationWindow.class); public MobileUiAutomationWindow(MobileAutomationService service, WorkUtilsProperties properties, CountDownLatch closeLatch) { this.service = service; @@ -65,6 +91,12 @@ public class MobileUiAutomationWindow { stage.show(); } + /** 窗口首次显示时执行初始化 */ + public void onFirstShow() { + // 自动检测设备并恢复上次选择 + refreshDevices(); + } + public Parent getContentPane() { return stage.getScene().getRoot(); } @@ -75,7 +107,10 @@ public class MobileUiAutomationWindow { private void buildUI() { stage.setTitle("手机UI自动化"); - stage.setOnHidden(e -> closeLatch.countDown()); + stage.setOnHidden(e -> { + stopScreenshotLoop(); + closeLatch.countDown(); + }); BorderPane root = new BorderPane(); root.setPadding(new Insets(Theme.S3)); @@ -102,21 +137,43 @@ public class MobileUiAutomationWindow { initEnvButton = StyledButton.primary("初始化环境", Theme.bold(properties.getFontButton())); Button envButton = StyledButton.secondary("检查环境", Theme.base(properties.getFontButton())); Button devicesButton = StyledButton.secondary("刷新设备", Theme.base(properties.getFontButton())); - Button casesButton = StyledButton.secondary("打开用例目录", Theme.base(properties.getFontButton())); - Button reportsButton = StyledButton.secondary("打开报告目录", Theme.base(properties.getFontButton())); + + // 辅助工具状态指示 + toolsStatusLabel = new Label(""); + toolsStatusLabel.setFont(Theme.base(properties.getFontSmall())); + toolsStatusLabel.setTextFill(Theme.TEXT_SECONDARY); downloadEnvButton.setOnAction(e -> openPythonDownload()); initEnvButton.setOnAction(e -> initializeEnvironment()); envButton.setOnAction(e -> checkEnvironment()); devicesButton.setOnAction(e -> refreshDevices()); - casesButton.setOnAction(e -> openPath(service.getCasesDir())); - reportsButton.setOnAction(e -> openPath(service.getReportsDir())); deviceCombo = new ComboBox<>(); deviceCombo.setPrefWidth(260); deviceCombo.setPromptText("选择设备"); + deviceCombo.setOnAction(e -> onDeviceSelected()); - HBox bar = new HBox(Theme.S2, title, new Region(), deviceCombo, downloadEnvButton, initEnvButton, envButton, devicesButton, casesButton, reportsButton); + // 设备连接状态指示 + deviceStatusLabel = new Label("⚪ 未检测"); + deviceStatusLabel.setFont(Theme.base(properties.getFontSmall())); + deviceStatusLabel.setTextFill(Theme.TEXT_SECONDARY); + + // 环境设置按钮折叠到 MenuButton 中 + MenuButton envMenuButton = new MenuButton("环境设置"); + envMenuButton.setFont(Theme.base(properties.getFontButton())); + envMenuButton.setStyle("-fx-background-color: transparent; -fx-border-color: " + Theme.toWeb(Theme.BORDER) + "; -fx-border-radius: 4; -fx-padding: 4 8;"); + MenuItem downloadItem = new MenuItem("下载 Python 3.11"); + MenuItem initItem = new MenuItem("初始化环境"); + MenuItem checkItem = new MenuItem("检查环境"); + installToolsItem = new MenuItem("安装辅助工具"); + downloadItem.setOnAction(e -> openPythonDownload()); + initItem.setOnAction(e -> initializeEnvironment()); + checkItem.setOnAction(e -> checkEnvironment()); + installToolsItem.setOnAction(e -> installAssistTools()); + envMenuButton.getItems().addAll(downloadItem, initItem, checkItem, new SeparatorMenuItem(), installToolsItem); + + HBox bar = new HBox(Theme.S2, title, new Region(), deviceStatusLabel, deviceCombo, toolsStatusLabel, + envMenuButton, devicesButton); HBox.setHgrow(bar.getChildren().get(1), Priority.ALWAYS); bar.setAlignment(Pos.CENTER_LEFT); bar.setPadding(new Insets(0, 0, Theme.S3, 0)); @@ -134,52 +191,65 @@ public class MobileUiAutomationWindow { } private Parent buildRunPane() { - packageField = new TextField(); - packageField.setPromptText("com.example.app"); - activityField = new TextField(); - activityField.setPromptText(".MainActivity，可选"); - caseList = new ListView<>(); + caseList.getStyleClass().add("compact-list"); + caseList.setFixedCellSize(28); caseList.setCellFactory(list -> new ListCell<>() { @Override protected void updateItem(Path item, boolean empty) { super.updateItem(item, empty); - setText(empty || item == null ? null : item.getFileName().toString()); + if (empty || item == null) { + setText(null); + setGraphic(null); + setTooltip(null); + return; + } + CasePreview preview = parseCasePreview(item); + setText(preview.name + " (" + preview.steps + "步)"); + setTooltip(new Tooltip(preview.description)); + if (item.toString().endsWith(".py") || item.toString().endsWith(".air")) { + setText(preview.name + " [Python脚本]"); + } } }); + // 右键菜单 + ContextMenu contextMenu = new ContextMenu(); + MenuItem renameItem = new MenuItem("重命名"); + MenuItem openItem = new MenuItem("打开文件所在位置"); + MenuItem editItem = new MenuItem("用记事本编辑"); + MenuItem deleteItem = new MenuItem("删除"); + renameItem.setOnAction(e -> renameSelectedCase()); + openItem.setOnAction(e -> openCaseFileLocation()); + editItem.setOnAction(e -> editCaseFile()); + deleteItem.setOnAction(e -> deleteSelectedCase()); + contextMenu.getItems().addAll(renameItem, new SeparatorMenuItem(), openItem, editItem, new SeparatorMenuItem(), deleteItem); + caseList.setContextMenu(contextMenu); Button refreshCasesButton = StyledButton.secondary("刷新用例", Theme.base(properties.getFontButton())); Button importCaseButton = StyledButton.secondary("导入用例", Theme.base(properties.getFontButton())); - runButton = StyledButton.primary("运行", Theme.bold(properties.getFontButton())); - stopButton = StyledButton.danger("停止", Theme.base(properties.getFontButton())); - stopButton.setDisable(true); + // 运行/停止合并按钮 + runStopButton = StyledButton.primary("运行", Theme.bold(properties.getFontButton())); + runStopButton.setPrefWidth(80); refreshCasesButton.setOnAction(e -> refreshCases()); importCaseButton.setOnAction(e -> importCase()); - runButton.setOnAction(e -> runSelectedCase()); - stopButton.setOnAction(e -> stopCurrentRun()); + runStopButton.setOnAction(e -> { + if (isRunning) { + stopCurrentRun(); + } else { + runSelectedCase(); + } + }); - GridPane form = new GridPane(); - form.setHgap(Theme.S2); - form.setVgap(Theme.S2); - form.add(new Label("Package"), 0, 0); - form.add(packageField, 1, 0); - form.add(new Label("Activity"), 0, 1); - form.add(activityField, 1, 1); - ColumnConstraints c0 = new ColumnConstraints(); - c0.setMinWidth(80); - ColumnConstraints c1 = new ColumnConstraints(); - c1.setHgrow(Priority.ALWAYS); - form.getColumnConstraints().addAll(c0, c1); - - HBox actions = new HBox(Theme.S2, refreshCasesButton, importCaseButton, new Region(), runButton, stopButton); + HBox actions = new HBox(Theme.S2, refreshCasesButton, importCaseButton, new Region(), runStopButton); HBox.setHgrow(actions.getChildren().get(2), Priority.ALWAYS); actions.setAlignment(Pos.CENTER_LEFT); - VBox left = new VBox(Theme.S2, form, actions, new Label("用例列表"), caseList); + VBox left = new VBox(Theme.S2, actions, new Label("用例列表"), caseList); VBox.setVgrow(caseList, Priority.ALWAYS); left.setPadding(new Insets(Theme.S3)); left.setPrefWidth(360); + left.setMinWidth(0); logArea = new TextArea(); logArea.setEditable(false); @@ -187,16 +257,75 @@ public class MobileUiAutomationWindow { logArea.setFont(Theme.mono(properties.getFontMono())); logArea.getStyleClass().add(Theme.CSS_LOG_AREA); - SplitPane split = new SplitPane(left, logArea); + // 执行历史记录 + historyList = new ListView<>(); + historyList.getStyleClass().add("compact-list"); + historyList.setFixedCellSize(28); + historyList.setCellFactory(list -> new ListCell<>() { + @Override + protected void updateItem(ExecutionHistoryEntry item, boolean empty) { + super.updateItem(item, empty); + if (empty || item == null) { + setText(null); + return; + } + setText(item.toString()); + setTooltip(new Tooltip("设备: " + item.device + "\n包名: " + (item.packageName != null ? item.packageName : "无"))); + } + }); + // 右键菜单：重命名、打开报告目录 + ContextMenu historyContextMenu = new ContextMenu(); + MenuItem renameReportItem = new MenuItem("重命名"); + MenuItem openReportDirItem = new MenuItem("打开报告目录"); + renameReportItem.setOnAction(e -> renameSelectedReport()); + openReportDirItem.setOnAction(e -> { + ExecutionHistoryEntry selected = historyList.getSelectionModel().getSelectedItem(); + if (selected != null && selected.reportDir != null) { + openPath(selected.reportDir); + } + }); + historyContextMenu.getItems().addAll(renameReportItem, new SeparatorMenuItem(), openReportDirItem); + historyList.setContextMenu(historyContextMenu); + // 双击历史记录也打开报告目录 + historyList.setOnMouseClicked(event -> { + if (event.getClickCount() == 2) { + ExecutionHistoryEntry selected = historyList.getSelectionModel().getSelectedItem(); + if (selected != null && selected.reportDir != null) { + openPath(selected.reportDir); + } + } + }); + Button refreshHistoryButton = StyledButton.secondary("刷新历史", Theme.base(properties.getFontSmall())); + refreshHistoryButton.setOnAction(e -> refreshHistory()); + Label historyLabel = new Label("执行历史"); + historyLabel.setFont(Theme.bold(properties.getFontSmall())); + HBox historyHeader = new HBox(Theme.S2, historyLabel, new Region(), refreshHistoryButton); + HBox.setHgrow(historyHeader.getChildren().get(1), Priority.ALWAYS); + historyHeader.setAlignment(Pos.CENTER_LEFT); + VBox historyBox = new VBox(Theme.S1, historyHeader, historyList); + historyBox.setPadding(new Insets(Theme.S2)); + VBox.setVgrow(historyList, Priority.ALWAYS); + + // 历史记录在上，日志在下，上下五五开 + historyBox.setMinHeight(0); // 允许拖动到最小 + logArea.setMinHeight(0); // 允许拖动到最小 + SplitPane verticalSplit = new SplitPane(historyBox, logArea); + verticalSplit.setOrientation(javafx.geometry.Orientation.VERTICAL); + verticalSplit.setDividerPositions(0.5); + verticalSplit.setMinWidth(300); + + SplitPane split = new SplitPane(left, verticalSplit); split.setDividerPositions(0.35); + + // 初始加载历史记录 + refreshHistory(); return split; } private Parent buildRecordPane() { phoneImageView = new ImageView(); phoneImageView.setPreserveRatio(true); - phoneImageView.setFitWidth(420); - phoneImageView.setFitHeight(720); + phoneImageView.setViewport(null); // 不使用视口裁剪 phoneImageView.setStyle("-fx-background-color: #111827; -fx-border-color: " + Theme.toWeb(Theme.BORDER) + "; -fx-border-width: 1;"); phoneImageView.setOnMouseClicked(event -> { if (event.getButton() == MouseButton.PRIMARY) { @@ -204,35 +333,157 @@ public class MobileUiAutomationWindow { } }); - Button startButton = StyledButton.primary("开始录制", Theme.bold(properties.getFontButton())); + // 开始录制/停止生成 合并按钮 + recordToggleButton = StyledButton.primary("开始录制", Theme.bold(properties.getFontButton())); + recordToggleButton.setPrefWidth(140); Button screenshotButton = StyledButton.secondary("刷新截图", Theme.base(properties.getFontButton())); Button backButton = StyledButton.secondary("返回", Theme.base(properties.getFontButton())); - Button stopButton = StyledButton.success("停止并生成用例", Theme.base(properties.getFontButton())); + Button undoButton = StyledButton.secondary("撤销上一步", Theme.base(properties.getFontButton())); recordInputField = new TextField(); recordInputField.setPromptText("输入文本后点击发送"); Button inputButton = StyledButton.secondary("发送输入", Theme.base(properties.getFontButton())); - startButton.setOnAction(e -> startRecording()); + recordToggleButton.setOnAction(e -> { + if (isRecording) { + stopRecording(); + } else { + startRecording(); + } + }); screenshotButton.setOnAction(e -> refreshRecordScreenshot()); backButton.setOnAction(e -> recordBack()); + undoButton.setOnAction(e -> undoLastStep()); inputButton.setOnAction(e -> recordInput()); - stopButton.setOnAction(e -> stopRecording()); recordStatusLabel = new Label("点击开始录制后，在手机截图上点击即可同步操作并记录步骤。"); recordStatusLabel.setTextFill(Theme.TEXT_SECONDARY); + // 录制时的 Package/Activity 配置 + recordPackageField = new TextField(); + recordPackageField.setPromptText("com.example.app（可选）"); + recordActivityField = new TextField(); + recordActivityField.setPromptText(".MainActivity（可选）"); + Button getAppButton = StyledButton.secondary("获取当前应用", Theme.base(properties.getFontSmall())); + getAppButton.setOnAction(e -> fetchCurrentApp()); + + GridPane appForm = new GridPane(); + appForm.setHgap(Theme.S2); + appForm.setVgap(Theme.S1); + appForm.add(new Label("Package"), 0, 0); + appForm.add(recordPackageField, 1, 0); + appForm.add(new Label("Activity"), 0, 1); + appForm.add(recordActivityField, 1, 1); + GridPane.setHgrow(recordPackageField, Priority.ALWAYS); + GridPane.setHgrow(recordActivityField, Priority.ALWAYS); + ColumnConstraints labelCol = new ColumnConstraints(); + labelCol.setMinWidth(60); + ColumnConstraints fieldCol = new ColumnConstraints(); + fieldCol.setHgrow(Priority.ALWAYS); + appForm.getColumnConstraints().addAll(labelCol, fieldCol); + + HBox appConfigBox = new HBox(Theme.S2, appForm, getAppButton); + HBox.setHgrow(appForm, Priority.ALWAYS); + appConfigBox.setAlignment(Pos.CENTER_LEFT); + + // 默认等待时间配置 + Label waitLabel = new Label("默认等待(ms):"); + waitLabel.setFont(Theme.base(properties.getFontSmall())); + defaultWaitField = new TextField("500"); + defaultWaitField.setPrefWidth(80); + defaultWaitField.setPromptText("500"); + // 监听输入变化，实时更新 service 的默认等待时间 + defaultWaitField.textProperty().addListener((obs, oldVal, newVal) -> { + try { + int waitMs = Integer.parseInt(newVal.trim()); + if (waitMs >= 100) { + service.setDefaultWaitTime(waitMs); + } + } catch (NumberFormatException ignored) { + // 忽略无效输入 + } + }); + HBox waitBox = new HBox(Theme.S2, waitLabel, defaultWaitField); + waitBox.setAlignment(Pos.CENTER_LEFT); + HBox inputBox = new HBox(Theme.S2, recordInputField, inputButton); HBox.setHgrow(recordInputField, Priority.ALWAYS); - VBox controls = new VBox(Theme.S2, startButton, screenshotButton, backButton, inputBox, stopButton, recordStatusLabel); + + // 录制步骤列表 + recordStepList = new ListView<>(); + recordStepList.setPrefHeight(150); + + // 右键菜单 + ContextMenu stepContextMenu = new ContextMenu(); + MenuItem editWaitItem = new MenuItem("编辑等待时间"); + MenuItem editPageFeatureItem = new MenuItem("设置页面特征"); + MenuItem identifyElementItem = new MenuItem("识别元素"); + MenuItem deleteStepItem = new MenuItem("删除此步骤"); + stepContextMenu.getItems().addAll(editWaitItem, editPageFeatureItem, new SeparatorMenuItem(), identifyElementItem, new SeparatorMenuItem(), deleteStepItem); + + recordStepList.setContextMenu(stepContextMenu); + recordStepList.setOnMousePressed(event -> { + if (event.isSecondaryButtonDown()) { + int index = recordStepList.getSelectionModel().getSelectedIndex(); + if (index >= 0 && index < recordedSteps.size()) { + stepContextMenu.show(recordStepList, event.getScreenX(), event.getScreenY()); + } + } + }); + + editWaitItem.setOnAction(e -> { + int index = recordStepList.getSelectionModel().getSelectedIndex(); + if (index >= 0) { + showEditWaitDialog(index); + } + }); + + editPageFeatureItem.setOnAction(e -> { + int index = recordStepList.getSelectionModel().getSelectedIndex(); + if (index >= 0) { + showEditPageFeatureDialog(index); + } + }); + + identifyElementItem.setOnAction(e -> { + int index = recordStepList.getSelectionModel().getSelectedIndex(); + if (index >= 0) { + identifyStepElement(index); + } + }); + + deleteStepItem.setOnAction(e -> { + int index = recordStepList.getSelectionModel().getSelectedIndex(); + if (index >= 0) { + deleteRecordedStep(index); + } + }); + + // 识别全部按钮 + Button identifyAllButton = StyledButton.secondary("识别全部", Theme.base(properties.getFontSmall())); + identifyAllButton.setOnAction(e -> identifyAllSteps()); + + Label stepsLabel = new Label("已录制步骤"); + stepsLabel.setFont(Theme.bold(properties.getFontSmall())); + stepsLabel.setTextFill(Theme.TEXT_SECONDARY); + HBox stepsHeader = new HBox(Theme.S2, stepsLabel, new Region(), identifyAllButton); + HBox.setHgrow(stepsHeader.getChildren().get(1), Priority.ALWAYS); + stepsHeader.setAlignment(Pos.CENTER_LEFT); + + VBox controls = new VBox(Theme.S2, recordToggleButton, appConfigBox, screenshotButton, backButton, undoButton, waitBox, inputBox, recordStatusLabel, stepsHeader, recordStepList); controls.setPadding(new Insets(Theme.S3)); - controls.setPrefWidth(330); + controls.setMinWidth(380); // 增加最小宽度，避免按钮被挤压 + controls.setMaxWidth(420); // 限制最大宽度 BorderPane preview = new BorderPane(phoneImageView); preview.setPadding(new Insets(Theme.S3)); preview.setStyle("-fx-background-color: #f8fafc;"); + preview.setMinSize(400, 600); // 设置最小尺寸，确保初始可见 + // 绑定尺寸，让 ImageView 自适应容器大小（保持比例，完整显示） + phoneImageView.fitWidthProperty().bind(preview.widthProperty().subtract(20)); + phoneImageView.fitHeightProperty().bind(preview.heightProperty().subtract(20)); HBox root = new HBox(Theme.S3, controls, preview); - HBox.setHgrow(preview, Priority.ALWAYS); + HBox.setHgrow(preview, Priority.ALWAYS); // preview 自动填充剩余空间 return root; } @@ -315,12 +566,123 @@ public class MobileUiAutomationWindow { Platform.runLater(() -> { deviceCombo.setItems(FXCollections.observableArrayList(devices)); if (!devices.isEmpty()) { - deviceCombo.getSelectionModel().select(0); + // 尝试恢复上次使用的设备 + String lastDevice = preferences.get(PREF_LAST_DEVICE, null); + if (lastDevice != null) { + for (int i = 0; i < devices.size(); i++) { + if (lastDevice.equals(devices.get(i).getSerial())) { + deviceCombo.getSelectionModel().select(i); + break; + } + } + } + if (deviceCombo.getSelectionModel().getSelectedIndex() < 0) { + deviceCombo.getSelectionModel().select(0); + } } + updateDeviceStatus(); setStatus("设备数量: " + devices.size(), false); }); } catch (Exception e) { - Platform.runLater(() -> setStatus("刷新设备失败: " + e.getMessage(), true)); + Platform.runLater(() -> { + updateDeviceStatus(); + setStatus("刷新设备失败: " + e.getMessage(), true); + }); + } + }); + } + + private void onDeviceSelected() { + DeviceInfo device = deviceCombo.getSelectionModel().getSelectedItem(); + if (device != null) { + preferences.put(PREF_LAST_DEVICE, device.getSerial()); + } + updateDeviceStatus(); + updateToolsStatus(); + } + + private void updateDeviceStatus() { + DeviceInfo device = deviceCombo.getSelectionModel().getSelectedItem(); + if (device == null) { + deviceStatusLabel.setText("🔴 未连接"); + deviceStatusLabel.setTextFill(Theme.DANGER); + return; + } + if (!device.isOnline()) { + deviceStatusLabel.setText("🔴 " + device.getSerial() + " 未授权"); + deviceStatusLabel.setTextFill(Theme.WARNING); + return; + } + // 异步验证设备是否真正在线 + runBackground(() -> { + boolean connected = service.isDeviceConnected(device.getSerial()); + Platform.runLater(() -> { + if (connected) { + deviceStatusLabel.setText("🟢 " + device.getSerial() + " 已连接"); + deviceStatusLabel.setTextFill(Theme.SUCCESS); + } else { + deviceStatusLabel.setText("🟡 " + device.getSerial() + " 检测中"); + deviceStatusLabel.setTextFill(Theme.WARNING); + } + }); + }); + } + + /** + * 更新辅助工具状态指示 + */ + private void updateToolsStatus() { + DeviceInfo device = deviceCombo.getSelectionModel().getSelectedItem(); + if (device == null) { + toolsStatusLabel.setText(""); + return; + } + boolean minicap = service.isMinicapInstalled(device.getSerial()); + boolean yosemite = service.isYosemiteInstalled(device.getSerial()); + if (minicap) { + toolsStatusLabel.setText("⚡ minicap"); + toolsStatusLabel.setTextFill(Theme.SUCCESS); + } else if (yosemite) { + toolsStatusLabel.setText("⚡ javacap"); + toolsStatusLabel.setTextFill(Theme.SUCCESS); + } else { + toolsStatusLabel.setText("⭕ adbcap"); + toolsStatusLabel.setTextFill(Theme.TEXT_SECONDARY); + } + } + + /** + * 安装辅助截图工具 + */ + private void installAssistTools() { + DeviceInfo device = deviceCombo.getSelectionModel().getSelectedItem(); + if (device == null) { + setStatus("请先选择设备", true); + return; + } + installToolsItem.setDisable(true); + setStatus("正在安装辅助工具...", false); + runBackground(() -> { + try { + service.installAssistTools(device.getSerial(), new AutomationCallback() { + @Override + public void onEvent(String type, String level, String message) { + Platform.runLater(() -> appendLog("[" + level + "] " + message)); + } + @Override + public void onDone(AutomationRunResult result) { + } + }); + Platform.runLater(() -> { + updateToolsStatus(); + setStatus("辅助工具安装完成", false); + installToolsItem.setDisable(false); + }); + } catch (Exception e) { + Platform.runLater(() -> { + setStatus("安装失败: " + e.getMessage(), true); + installToolsItem.setDisable(false); + }); } }); } @@ -332,10 +694,18 @@ public class MobileUiAutomationWindow { caseList.setItems(FXCollections.observableArrayList(service.listCases())); } + private void refreshHistory() { + if (historyList == null) { + return; + } + List entries = service.listExecutionHistory(50); + historyList.setItems(FXCollections.observableArrayList(entries)); + } + private void importCase() { FileChooser chooser = new FileChooser(); chooser.setTitle("导入自动化用例"); - chooser.getExtensionFilters().add(new FileChooser.ExtensionFilter("Automation Cases", "*.yaml", "*.yml", "*.py", "*.air")); + chooser.getExtensionFilters().add(new FileChooser.ExtensionFilter("Automation Cases", "*.json", "*.yaml", "*.yml", "*.py", "*.air")); File file = chooser.showOpenDialog(stage); if (file == null) { return; @@ -349,23 +719,150 @@ public class MobileUiAutomationWindow { } } + private void openCaseFileLocation() { + Path selected = caseList.getSelectionModel().getSelectedItem(); + if (selected == null) { + setStatus("请先选择要打开的用例", true); + return; + } + try { + SystemOpenHelper.revealInFileManager(selected.getParent()); + } catch (Exception e) { + setStatus("打开目录失败: " + e.getMessage(), true); + } + } + + private void editCaseFile() { + Path selected = caseList.getSelectionModel().getSelectedItem(); + if (selected == null) { + setStatus("请先选择要编辑的用例", true); + return; + } + try { + if (Desktop.isDesktopSupported() && Desktop.getDesktop().isSupported(Desktop.Action.EDIT)) { + Desktop.getDesktop().edit(selected.toFile()); + } else { + // 回退：用记事本打开 + new ProcessBuilder("notepad", selected.toAbsolutePath().toString()).start(); + } + } catch (Exception e) { + setStatus("编辑失败: " + e.getMessage(), true); + } + } + + private void renameSelectedCase() { + Path selected = caseList.getSelectionModel().getSelectedItem(); + if (selected == null) { + setStatus("请先选择要重命名的用例", true); + return; + } + String oldName = selected.getFileName().toString(); + if (oldName.endsWith(".json")) { + oldName = oldName.substring(0, oldName.length() - 5); + } + TextInputDialog dialog = new TextInputDialog(oldName); + dialog.setTitle("重命名用例"); + dialog.setHeaderText(null); + dialog.setContentText("新名称:"); + dialog.showAndWait().ifPresent(newName -> { + if (newName != null && !newName.trim().isEmpty()) { + try { + service.renameCase(selected, newName.trim()); + refreshCases(); + setStatus("已重命名: " + newName.trim(), false); + } catch (Exception e) { + setStatus("重命名失败: " + e.getMessage(), true); + } + } + }); + } + + private void renameSelectedReport() { + ExecutionHistoryEntry selected = historyList.getSelectionModel().getSelectedItem(); + if (selected == null || selected.reportDir == null) { + setStatus("请先选择要重命名的报告", true); + return; + } + String oldName = selected.reportDir.getFileName().toString(); + TextInputDialog dialog = new TextInputDialog(oldName); + dialog.setTitle("重命名报告"); + dialog.setHeaderText(null); + dialog.setContentText("新名称:"); + dialog.showAndWait().ifPresent(newName -> { + if (newName != null && !newName.trim().isEmpty()) { + try { + service.renameReport(selected.reportDir, newName.trim()); + refreshHistory(); + setStatus("已重命名: " + newName.trim(), false); + } catch (Exception e) { + setStatus("重命名失败: " + e.getMessage(), true); + } + } + }); + } + + private void deleteSelectedCase() { + Path selected = caseList.getSelectionModel().getSelectedItem(); + if (selected == null) { + setStatus("请先选择要删除的用例", true); + return; + } + Alert alert = new Alert(Alert.AlertType.CONFIRMATION); + alert.setTitle("确认删除"); + alert.setHeaderText("删除用例"); + alert.setContentText("确定要删除用例吗?\n" + selected.getFileName()); + alert.showAndWait().ifPresent(result -> { + if (result == javafx.scene.control.ButtonType.OK) { + try { + Files.deleteIfExists(selected); + refreshCases(); + setStatus("已删除用例: " + selected.getFileName(), false); + } catch (Exception e) { + setStatus("删除失败: " + e.getMessage(), true); + } + } + }); + } + private void runSelectedCase() { Path casePath = caseList.getSelectionModel().getSelectedItem(); if (casePath == null) { setStatus("请选择用例", true); return; } + + // 从用例文件中读取 Package/Activity + String packageName = ""; + String activityName = ""; + try { + com.alibaba.fastjson.JSONObject caseJson = parseCaseFile(casePath); + if (caseJson != null) { + com.alibaba.fastjson.JSONObject app = caseJson.getJSONObject("app"); + if (app != null) { + packageName = app.getString("package"); + if (packageName != null && !packageName.isEmpty()) { + appendLog("[INFO] 从用例文件读取到 Package: " + packageName); + } + activityName = app.getString("activity"); + if (activityName != null && !activityName.isEmpty()) { + appendLog("[INFO] 从用例文件读取到 Activity: " + activityName); + } + } + } + } catch (Exception e) { + appendLog("[WARN] 解析用例文件失败: " + e.getMessage()); + } + AutomationRunConfig config = new AutomationRunConfig(); DeviceInfo device = deviceCombo.getSelectionModel().getSelectedItem(); config.setDeviceSerial(device != null ? device.getSerial() : ""); - config.setPackageName(packageField.getText().trim()); - config.setActivityName(activityField.getText().trim()); + config.setPackageName(packageName != null ? packageName : ""); + config.setActivityName(activityName != null ? activityName : ""); config.setCasePath(casePath); config.setTimeoutSeconds(20); config.setScreenshotOnFail(true); - runButton.setDisable(true); - stopButton.setDisable(false); + setRunningState(true); try { currentRun = service.runCase(config, new AutomationCallback() { @Override @@ -376,16 +873,18 @@ public class MobileUiAutomationWindow { @Override public void onDone(AutomationRunResult result) { Platform.runLater(() -> { - runButton.setDisable(false); - stopButton.setDisable(true); + setRunningState(false); appendLog(result.getMessage()); setStatus(result.getMessage(), !result.isSuccess()); + if (result.getReportDir() != null) { + displayRunResult(result.getReportDir()); + } + refreshHistory(); }); } }); } catch (Exception e) { - runButton.setDisable(false); - stopButton.setDisable(true); + setRunningState(false); setStatus("运行失败: " + e.getMessage(), true); } } @@ -394,21 +893,63 @@ public class MobileUiAutomationWindow { if (currentRun != null) { service.stopRun(currentRun.getRunId()); appendLog("已请求停止运行"); - stopButton.setDisable(true); + setRunningState(false); + } + } + + /** 切换运行/停止按钮状态 */ + private void setRunningState(boolean running) { + isRunning = running; + if (running) { + runStopButton.setText("停止"); + runStopButton.setStyle("-fx-background-color: " + Theme.toWeb(Theme.DANGER) + "; -fx-text-fill: white; -fx-font-weight: bold; -fx-background-radius: 4;"); + } else { + runStopButton.setText("运行"); + runStopButton.setStyle("-fx-background-color: " + Theme.toWeb(Theme.PRIMARY) + "; -fx-text-fill: white; -fx-font-weight: bold; -fx-background-radius: 4;"); } } private void startRecording() { DeviceInfo device = deviceCombo.getSelectionModel().getSelectedItem(); try { - service.startRecording(device != null ? device.getSerial() : "", packageField.getText().trim()); + String pkg = recordPackageField != null ? recordPackageField.getText().trim() : ""; + service.startRecording(device != null ? device.getSerial() : "", pkg); + recordedSteps.clear(); + Platform.runLater(() -> { + recordStepList.setItems(FXCollections.observableArrayList(recordedSteps)); + }); + setRecordingState(true); setRecordStatus("录制已开始"); + startScreenshotLoop(); refreshRecordScreenshot(); } catch (Exception e) { setRecordStatus("开始录制失败: " + e.getMessage()); } } + /** + * 从设备获取当前前台应用包名并填入录制面板的 Package 字段 + */ + private void fetchCurrentApp() { + DeviceInfo device = deviceCombo.getSelectionModel().getSelectedItem(); + if (device == null) { + setRecordStatus("请先选择设备"); + return; + } + setRecordStatus("正在获取当前应用..."); + runBackground(() -> { + String pkg = service.getCurrentForegroundApp(device.getSerial()); + Platform.runLater(() -> { + if (pkg != null && !pkg.isEmpty()) { + recordPackageField.setText(pkg); + setRecordStatus("已获取当前应用: " + pkg); + } else { + setRecordStatus("未检测到前台应用（可能是桌面启动器），请先打开目标应用"); + } + }); + }); + } + private void refreshRecordScreenshot() { runBackground(() -> { try { @@ -423,6 +964,62 @@ public class MobileUiAutomationWindow { }); } + // ==================== 实时截图轮询 ==================== + + private void startScreenshotLoop() { + stopScreenshotLoop(); + screenshotRunning = true; + screenshotThread = new Thread(() -> { + int errorCount = 0; + while (screenshotRunning) { + try { + if (service.isRecording()) { + long start = System.currentTimeMillis(); + Path screenshot = service.captureRecordingScreenshot(); + long elapsed = System.currentTimeMillis() - start; + Platform.runLater(() -> { + try { + phoneImageView.setImage(new Image(screenshot.toUri().toString(), false)); + } catch (Exception ignored) { + } + }); + errorCount = 0; + // 自适应等待：截图耗时 + 500ms 间隔，避免命令堆积 + long waitTime = Math.max(500, 2000 - elapsed); + Thread.sleep(waitTime); + } else { + Thread.sleep(500); + } + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + break; + } catch (Exception e) { + errorCount++; + if (errorCount > 5) { + Platform.runLater(() -> setRecordStatus("截图轮询异常: " + e.getMessage())); + break; + } + try { + Thread.sleep(1000); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + break; + } + } + } + }, "screenshot-loop"); + screenshotThread.setDaemon(true); + screenshotThread.start(); + } + + private void stopScreenshotLoop() { + screenshotRunning = false; + if (screenshotThread != null) { + screenshotThread.interrupt(); + screenshotThread = null; + } + } + private void recordTap(double x, double y) { Image image = phoneImageView.getImage(); if (image == null) { @@ -440,9 +1037,15 @@ public class MobileUiAutomationWindow { } service.recordTap(localX, localY, area.width, area.height, (int) image.getWidth(), (int) image.getHeight()); + int screenX = (int) Math.round(localX / area.width * image.getWidth()); + int screenY = (int) Math.round(localY / area.height * image.getHeight()); Platform.runLater(() -> { - setRecordStatus("已记录点击: " + Math.round(x) + ", " + Math.round(y)); - refreshRecordScreenshot(); + setRecordStatus("已记录点击: " + Math.round(x) + ", " + Math.round(y) + " (" + screenX + "," + screenY + ")"); + addRecordedStep("点击 (" + screenX + "," + screenY + ")"); + // 操作后自动刷新截图 + if (!screenshotRunning) { + refreshRecordScreenshot(); + } }); } catch (Exception e) { Platform.runLater(() -> setRecordStatus("记录点击失败: " + e.getMessage())); @@ -452,13 +1055,20 @@ public class MobileUiAutomationWindow { private void recordInput() { String text = recordInputField.getText(); + if (text == null || text.isEmpty()) { + setRecordStatus("请输入内容"); + return; + } runBackground(() -> { try { service.recordInput(text); Platform.runLater(() -> { recordInputField.clear(); - setRecordStatus("已记录输入"); - refreshRecordScreenshot(); + setRecordStatus("已记录输入: " + text); + addRecordedStep("输入文本: " + text); + if (!screenshotRunning) { + refreshRecordScreenshot(); + } }); } catch (Exception e) { Platform.runLater(() -> setRecordStatus("输入失败: " + e.getMessage())); @@ -472,7 +1082,10 @@ public class MobileUiAutomationWindow { service.recordBack(); Platform.runLater(() -> { setRecordStatus("已记录返回"); - refreshRecordScreenshot(); + addRecordedStep("返回 (BACK)"); + if (!screenshotRunning) { + refreshRecordScreenshot(); + } }); } catch (Exception e) { Platform.runLater(() -> setRecordStatus("返回失败: " + e.getMessage())); @@ -480,17 +1093,396 @@ public class MobileUiAutomationWindow { }); } + private void addRecordedStep(String step) { + int waitMs = service.getDefaultWaitTime(); + recordedSteps.add((recordedSteps.size() + 1) + ". " + step + " (wait:" + waitMs + "ms)"); + Platform.runLater(() -> { + recordStepList.setItems(FXCollections.observableArrayList(recordedSteps)); + recordStepList.scrollTo(recordedSteps.size() - 1); + }); + } + private void stopRecording() { + stopScreenshotLoop(); + setRecordingState(false); try { - Path yaml = service.stopRecording(); + Path caseFile = service.stopRecording(); refreshCases(); - setRecordStatus("已生成用例: " + yaml.getFileName()); - setStatus("录制用例已生成: " + yaml.toAbsolutePath(), false); + setRecordStatus("✅ 用例已生成: " + caseFile.getFileName()); + setStatus("🎉 录制用例已生成: " + caseFile.getFileName(), false); + + Alert alert = new Alert(Alert.AlertType.INFORMATION); + alert.setTitle("用例生成成功"); + alert.setHeaderText(null); + alert.setContentText("用例文件已生成:\n" + caseFile.getFileName() + "\n\n位置: " + caseFile.getParent()); + ButtonType openDirButton = new ButtonType("打开目录", ButtonBar.ButtonData.OK_DONE); + ButtonType closeButton = new ButtonType("关闭", ButtonBar.ButtonData.CANCEL_CLOSE); + alert.getButtonTypes().setAll(openDirButton, closeButton); + Optional result = alert.showAndWait(); + if (result.isPresent() && result.get() == openDirButton) { + openPath(caseFile.getParent()); + } + + Platform.runLater(() -> { + if (caseList != null) { + for (Path p : caseList.getItems()) { + if (p.getFileName().toString().equals(caseFile.getFileName().toString())) { + caseList.getSelectionModel().select(p); + break; + } + } + } + }); } catch (Exception e) { - setRecordStatus("停止录制失败: " + e.getMessage()); + setRecordStatus("❌ 停止录制失败: " + e.getMessage()); } } + /** 切换开始录制/停止生成按钮状态 */ + private void setRecordingState(boolean recording) { + isRecording = recording; + if (recording) { + recordToggleButton.setText("停止并生成用例"); + recordToggleButton.setStyle("-fx-background-color: " + Theme.toWeb(Theme.SUCCESS) + "; -fx-text-fill: white; -fx-font-weight: bold; -fx-background-radius: 4;"); + } else { + recordToggleButton.setText("开始录制"); + recordToggleButton.setStyle("-fx-background-color: " + Theme.toWeb(Theme.PRIMARY) + "; -fx-text-fill: white; -fx-font-weight: bold; -fx-background-radius: 4;"); + } + } + + private void undoLastStep() { + try { + String removed = service.undoLastStep(); + if (removed != null) { + // 同步 UI 列表 + if (!recordedSteps.isEmpty()) { + recordedSteps.remove(recordedSteps.size() - 1); + Platform.runLater(() -> { + recordStepList.setItems(FXCollections.observableArrayList(recordedSteps)); + }); + } + setRecordStatus("已撤销上一步"); + } else { + setRecordStatus("没有可撤销的步骤"); + } + } catch (Exception e) { + setRecordStatus("撤销失败: " + e.getMessage()); + } + } + + /** + * 显示编辑等待时间的对话框 + */ + private void showEditWaitDialog(int index) { + Dialog dialog = new Dialog<>(); + dialog.setTitle("编辑等待时间"); + dialog.setHeaderText("修改步骤 #" + (index + 1) + " 的等待时间（毫秒）"); + + TextField waitField = new TextField(); + waitField.setPromptText("例如：1000"); + waitField.setText("500"); + + ButtonType okButton = new ButtonType("确定", ButtonBar.ButtonData.OK_DONE); + ButtonType cancelButton = new ButtonType("取消", ButtonBar.ButtonData.CANCEL_CLOSE); + dialog.getDialogPane().getButtonTypes().addAll(okButton, cancelButton); + dialog.getDialogPane().setContent(new VBox(Theme.S2, new Label("等待时间(ms):"), waitField)); + + dialog.setResultConverter(button -> { + if (button == okButton) { + try { + return Integer.parseInt(waitField.getText().trim()); + } catch (NumberFormatException ex) { + showAlert("错误", "请输入有效的数字"); + return null; + } + } + return null; + }); + + Optional result = dialog.showAndWait(); + if (result.isPresent()) { + int waitMs = result.get(); + boolean success = service.updateStepWaitTime(index, waitMs); + if (success) { + // 重新构建步骤显示文本（保留页面特征标记） + String oldStep = recordedSteps.get(index); + String baseStep = oldStep.replaceAll(" \\(wait:\\s*\\d+ms\\)", "").replaceAll(" \\[page:[^\\]]+\\]", ""); + recordedSteps.set(index, baseStep + " (wait:" + waitMs + "ms)"); + refreshStepListDisplay(); + setRecordStatus("已更新步骤 #" + (index + 1) + " 的等待时间为 " + waitMs + "ms"); + } else { + showAlert("错误", "更新等待时间失败，索引无效"); + } + } + } + + /** + * 设置指定步骤的页面特征 + */ + private void showEditPageFeatureDialog(int index) { + Dialog dialog = new Dialog<>(); + dialog.setTitle("设置页面特征"); + dialog.setHeaderText("为步骤 #" + (index + 1) + " 设置页面特征名称\n用于执行时验证当前页面是否匹配"); + + TextField featureField = new TextField(); + featureField.setPromptText("例如：login-page, ad-popup, home-page"); + + // 尝试获取当前已有的页面特征 + String existingFeature = service.getStepPageFeature(index); + if (existingFeature != null) { + featureField.setText(existingFeature); + } + + ButtonType okButton = new ButtonType("确定", ButtonBar.ButtonData.OK_DONE); + ButtonType cancelButton = new ButtonType("取消", ButtonBar.ButtonData.CANCEL_CLOSE); + dialog.getDialogPane().getButtonTypes().addAll(okButton, cancelButton); + + VBox content = new VBox(Theme.S2, + new Label("页面特征名称:"), + featureField, + new Label("提示：为空则清除页面特征")); + content.setPadding(new Insets(Theme.S3)); + dialog.getDialogPane().setContent(content); + + dialog.setResultConverter(button -> { + if (button == okButton) { + return featureField.getText().trim(); + } + return null; + }); + + Optional result = dialog.showAndWait(); + if (result != null) { + String featureName = result.get(); + // 如果为空字符串，表示清除页面特征 + boolean success = service.setStepPageFeature(index, featureName.isEmpty() ? null : featureName, null); + if (success) { + refreshStepListDisplay(); + if (featureName.isEmpty()) { + setRecordStatus("已清除步骤 #" + (index + 1) + " 的页面特征"); + } else { + setRecordStatus("已设置步骤 #" + (index + 1) + " 的页面特征: " + featureName); + } + } else { + showAlert("错误", "设置页面特征失败，索引无效"); + } + } + } + + /** + * 刷新步骤列表显示，包含页面特征信息 + */ + private void refreshStepListDisplay() { + for (int i = 0; i < recordedSteps.size(); i++) { + String feature = service.getStepPageFeature(i); + String stepText = recordedSteps.get(i); + // 移除已有的页面特征标记 + stepText = stepText.replaceAll(" \\[page:[^\\]]+\\]", ""); + if (feature != null && !feature.isEmpty()) { + stepText = stepText + " [page:" + feature + "]"; + } + recordedSteps.set(i, stepText); + } + Platform.runLater(() -> { + recordStepList.setItems(FXCollections.observableArrayList(recordedSteps)); + }); + } + + /** + * 删除指定的录制步骤 + */ + private void deleteRecordedStep(int index) { + Alert alert = new Alert(Alert.AlertType.CONFIRMATION); + alert.setTitle("确认删除"); + alert.setHeaderText("删除步骤 #" + (index + 1)); + alert.setContentText("确定要删除此步骤吗？此操作不可恢复。"); + + Optional result = alert.showAndWait(); + if (result.isPresent() && result.get() == ButtonType.OK) { + boolean success = service.deleteStep(index); + if (success) { + recordedSteps.remove(index); + refreshStepListDisplay(); + setRecordStatus("已删除步骤 #" + (index + 1)); + } else { + showAlert("错误", "删除步骤失败，索引无效"); + } + } + } + + /** + * 识别指定步骤的 UI 元素（单个步骤） + */ + private void identifyStepElement(int index) { + DeviceInfo device = deviceCombo.getSelectionModel().getSelectedItem(); + if (device == null) { + setRecordStatus("请先选择设备"); + return; + } + // 获取步骤信息 + JSONObject stepInfo = service.getStepInfo(index); + if (stepInfo == null) { + setRecordStatus("无法获取步骤信息"); + return; + } + if (!"tap".equals(stepInfo.getString("action"))) { + setRecordStatus("只有点击步骤可以识别元素"); + return; + } + JSONObject target = stepInfo.getJSONObject("target"); + if (target == null) { + setRecordStatus("步骤缺少目标信息"); + return; + } + int x = target.getIntValue("x"); + int y = target.getIntValue("y"); + + setRecordStatus("正在识别元素 (" + x + "," + y + ")，请稍候..."); + runBackground(() -> { + try { + List elements = service.identifyElement(device.getSerial(), x, y); + Platform.runLater(() -> { + if (elements.isEmpty()) { + setRecordStatus("未找到匹配的 UI 元素，可能页面已变化"); + return; + } + showElementChooserDialog(index, x, y, elements); + }); + } catch (Exception e) { + Platform.runLater(() -> setRecordStatus("识别失败: " + e.getMessage())); + } + }); + } + + /** + * 批量识别所有点击步骤的 UI 元素 + */ + private void identifyAllSteps() { + DeviceInfo device = deviceCombo.getSelectionModel().getSelectedItem(); + if (device == null) { + setRecordStatus("请先选择设备"); + return; + } + if (!service.isRecording()) { + setRecordStatus("请先开始录制后再识别"); + return; + } + setRecordStatus("正在批量识别元素..."); + runBackground(() -> { + int identified = 0; + int total = 0; + for (int i = 0; i < recordedSteps.size(); i++) { + JSONObject stepInfo = service.getStepInfo(i); + if (stepInfo == null || !"tap".equals(stepInfo.getString("action"))) { + continue; + } + total++; + JSONObject target = stepInfo.getJSONObject("target"); + if (target == null) continue; + int x = target.getIntValue("x"); + int y = target.getIntValue("y"); + try { + List elements = service.identifyElement(device.getSerial(), x, y); + if (!elements.isEmpty()) { + JSONObject best = elements.get(0); // 面积最小的元素 + if (service.updateStepElement(i, best)) { + identified++; + final int stepIdx = i; + final JSONObject elem = best; + Platform.runLater(() -> updateStepDisplayText(stepIdx, elem)); + } + } + } catch (Exception e) { + final int stepIdx = i; + Platform.runLater(() -> appendLog("[WARN] 步骤 #" + (stepIdx + 1) + " 识别失败: " + e.getMessage())); + } + } + final int id = identified; + final int tt = total; + Platform.runLater(() -> setRecordStatus("批量识别完成: " + id + "/" + tt + " 个步骤已识别元素")); + }); + } + + /** + * 显示元素选择对话框，让用户选择最合适的元素 + */ + private void showElementChooserDialog(int stepIndex, int x, int y, List elements) { + ChoiceDialog dialog = new ChoiceDialog<>(); + dialog.setTitle("选择 UI 元素"); + dialog.setHeaderText("点击位置 (" + x + ", " + y + ") 找到 " + elements.size() + " 个元素：\n请选择最合适的元素进行定位"); + + List choices = new ArrayList<>(); + for (JSONObject elem : elements) { + String resourceId = elem.getString("resourceId"); + String text = elem.getString("text"); + String contentDesc = elem.getString("contentDesc"); + String className = elem.getString("className"); + // 简化类名 + if (className != null && className.contains(".")) { + className = className.substring(className.lastIndexOf('.') + 1); + } + StringBuilder sb = new StringBuilder(); + if (resourceId != null && !resourceId.isEmpty()) { + sb.append("id:").append(resourceId); + } else if (contentDesc != null && !contentDesc.isEmpty()) { + sb.append("desc:").append(contentDesc); + } else if (text != null && !text.isEmpty()) { + sb.append("text:").append(text); + } else { + sb.append("(无标识)"); + } + sb.append(" [").append(className).append("]"); + choices.add(sb.toString()); + } + dialog.getItems().addAll(choices); + dialog.setSelectedItem(choices.get(0)); + + Optional result = dialog.showAndWait(); + if (result.isPresent()) { + int selectedIndex = choices.indexOf(result.get()); + if (selectedIndex >= 0) { + JSONObject selected = elements.get(selectedIndex); + boolean updated = service.updateStepElement(stepIndex, selected); + if (updated) { + updateStepDisplayText(stepIndex, selected); + setRecordStatus("✅ 步骤 #" + (stepIndex + 1) + " 已识别: " + result.get()); + } else { + setRecordStatus("❗ 该元素无可识别属性，仅支持坐标点击"); + } + } + } + } + + /** + * 更新步骤列表显示文本（添加元素信息） + */ + private void updateStepDisplayText(int index, JSONObject element) { + if (index < 0 || index >= recordedSteps.size()) return; + String oldText = recordedSteps.get(index); + // 如果已经有 -> 标记，先移除旧标记 + int arrowIdx = oldText.indexOf(" -> "); + if (arrowIdx >= 0) { + oldText = oldText.substring(0, arrowIdx); + } + String resourceId = element.getString("resourceId"); + String text = element.getString("text"); + String contentDesc = element.getString("contentDesc"); + String label; + if (resourceId != null && !resourceId.isEmpty()) { + // 简化 resource-id，取冒号后的部分 + int colonIdx = resourceId.indexOf(':'); + label = colonIdx >= 0 ? resourceId.substring(colonIdx + 1) : resourceId; + } else if (contentDesc != null && !contentDesc.isEmpty()) { + label = contentDesc; + } else if (text != null && !text.isEmpty()) { + label = text; + } else { + label = "(无标识)"; + } + recordedSteps.set(index, oldText + " -> [" + label + "]"); + recordStepList.setItems(FXCollections.observableArrayList(recordedSteps)); + } + private void appendLog(String message) { if (logArea != null) { logArea.appendText(message + System.lineSeparator()); @@ -516,6 +1508,165 @@ public class MobileUiAutomationWindow { } } + /** + * 显示提示对话框 + */ + private void showAlert(String title, String message) { + Alert alert = new Alert(Alert.AlertType.INFORMATION); + alert.setTitle(title); + alert.setHeaderText(null); + alert.setContentText(message); + alert.showAndWait(); + } + + // ==================== 用例预览解析 ==================== + + private CasePreview parseCasePreview(Path path) { + String name = path.getFileName().toString(); + String lower = name.toLowerCase(); + if (lower.endsWith(".py") || lower.endsWith(".air")) { + return new CasePreview(name, 0, "Python 脚本用例"); + } + try { + List lines = Files.readAllLines(path, java.nio.charset.StandardCharsets.UTF_8); + String yamlName = ""; + int steps = 0; + StringBuilder desc = new StringBuilder(); + boolean inSteps = false; + + // JSON 格式解析 + if (lower.endsWith(".json")) { + String content = String.join("\n", lines); + com.alibaba.fastjson.JSONObject json = com.alibaba.fastjson.JSON.parseObject(content); + if (json != null) { + yamlName = json.getString("name"); + com.alibaba.fastjson.JSONArray stepsArray = json.getJSONArray("steps"); + if (stepsArray != null) { + steps = stepsArray.size(); + } + com.alibaba.fastjson.JSONObject app = json.getJSONObject("app"); + if (app != null && app.containsKey("package")) { + String pkg = app.getString("package"); + if (pkg != null && !pkg.isEmpty()) { + desc.append("APP: ").append(pkg).append(" "); + } + } + } + } else { + // YAML 格式解析（原有逻辑） + for (String line : lines) { + String trimmed = line.trim(); + if (trimmed.startsWith("name:")) { + yamlName = trimmed.substring(5).trim(); + } + if (trimmed.startsWith("steps:")) { + inSteps = true; + continue; + } + if (inSteps && trimmed.startsWith("- name:")) { + steps++; + } + if (trimmed.startsWith("package:")) { + desc.append("APP: ").append(trimmed.substring(8).trim()).append(" "); + } + } + } + + if (steps > 0) { + desc.append("| 共 ").append(steps).append(" 步"); + } + return new CasePreview(yamlName.isEmpty() ? name : yamlName, steps, desc.toString().trim()); + } catch (Exception e) { + return new CasePreview(name, 0, "无法解析"); + } + } + + private static class CasePreview { + final String name; + final int steps; + final String description; + + CasePreview(String name, int steps, String description) { + this.name = name; + this.steps = steps; + this.description = description; + } + } + + /** + * 解析用例文件，返回 JSONObject（支持 JSON 和 YAML） + */ + private com.alibaba.fastjson.JSONObject parseCaseFile(Path casePath) throws Exception { + String fileName = casePath.getFileName().toString().toLowerCase(); + + // JSON 格式 + if (fileName.endsWith(".json")) { + String content = Files.readString(casePath, java.nio.charset.StandardCharsets.UTF_8); + return com.alibaba.fastjson.JSON.parseObject(content); + } + + // YAML 格式（需要第三方库支持，这里简化处理） + if (fileName.endsWith(".yaml") || fileName.endsWith(".yml")) { + // TODO: 如果项目中有 SnakeYAML 依赖，可以添加 YAML 解析 + // 目前仅支持从文件名推断基本信息 + throw new UnsupportedOperationException("暂不支持直接解析 YAML 文件内容，请转换为 JSON 格式"); + } + + return null; + } + + // ==================== 运行结果可视化 ==================== + + private void displayRunResult(Path reportDir) { + try { + Path resultJson = reportDir.resolve("result.json"); + if (!Files.exists(resultJson)) { + appendLog("[WARN] 未找到 result.json: " + resultJson); + return; + } + String json = Files.readString(resultJson, java.nio.charset.StandardCharsets.UTF_8); + com.alibaba.fastjson.JSONObject obj = com.alibaba.fastjson.JSON.parseObject(json); + if (obj == null) { + appendLog("[WARN] 无法解析 result.json"); + return; + } + String caseName = obj.getString("caseName"); + Boolean success = obj.getBoolean("success"); + appendLog(""); + appendLog("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"); + appendLog("📋 用例: " + (caseName != null ? caseName : "未知")); + appendLog("🎯 结果: " + (Boolean.TRUE.equals(success) ? "✅ 通过" : "❌ 失败")); + appendLog("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"); + + com.alibaba.fastjson.JSONArray steps = obj.getJSONArray("steps"); + if (steps != null) { + appendLog("📊 步骤明细:"); + for (int i = 0; i < steps.size(); i++) { + com.alibaba.fastjson.JSONObject step = steps.getJSONObject(i); + String name = step.getString("name"); + String status = step.getString("status"); + Integer elapsed = step.getInteger("elapsedMs"); + String icon = "PASS".equalsIgnoreCase(status) ? "✅" : "❌"; + String time = elapsed != null ? " (" + elapsed + "ms)" : ""; + appendLog(" " + icon + " " + (i + 1) + ". " + name + time); + if (!"PASS".equalsIgnoreCase(status)) { + String msg = step.getString("message"); + if (msg != null && !msg.isEmpty()) { + appendLog(" ⚠️ " + msg); + } + String screenshot = step.getString("screenshot"); + if (screenshot != null && !screenshot.isEmpty()) { + appendLog(" 📸 截图: " + screenshot); + } + } + } + appendLog("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"); + } + } catch (Exception e) { + appendLog("[WARN] 结果可视化失败: " + e.getMessage()); + } + } + private void runBackground(Runnable runnable) { Thread thread = new Thread(runnable, "mobile-ui-auto-ui-worker"); thread.setDaemon(true); @@ -523,14 +1674,16 @@ public class MobileUiAutomationWindow { } private DisplayArea displayArea(Image image) { - double fitWidth = phoneImageView.getFitWidth(); - double fitHeight = phoneImageView.getFitHeight(); + // 使用 ImageView 的实际显示尺寸，而非 fit 尺寸 + // 因为 ImageView 在容器中可能被拉伸 + double actualWidth = phoneImageView.getBoundsInLocal().getWidth(); + double actualHeight = phoneImageView.getBoundsInLocal().getHeight(); double imageWidth = image.getWidth(); double imageHeight = image.getHeight(); - double scale = Math.min(fitWidth / imageWidth, fitHeight / imageHeight); + double scale = Math.min(actualWidth / imageWidth, actualHeight / imageHeight); double width = imageWidth * scale; double height = imageHeight * scale; - return new DisplayArea((fitWidth - width) / 2, (fitHeight - height) / 2, width, height); + return new DisplayArea((actualWidth - width) / 2, (actualHeight - height) / 2, width, height); } private static class DisplayArea { diff --git a/src/main/java/com/example/workutils/biz/mobileuiauto/service/MobileAutomationService.java b/src/main/java/com/example/workutils/biz/mobileuiauto/service/MobileAutomationService.java index a3af54f..b90c397 100644 --- a/src/main/java/com/example/workutils/biz/mobileuiauto/service/MobileAutomationService.java +++ b/src/main/java/com/example/workutils/biz/mobileuiauto/service/MobileAutomationService.java @@ -9,6 +9,7 @@ import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; import java.io.*; import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -24,9 +25,12 @@ import java.util.concurrent.TimeUnit; public class MobileAutomationService { private static final DateTimeFormatter RUN_ID_FMT = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss"); + private static final DateTimeFormatter CASE_NAME_FMT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH-mm"); + private static final DateTimeFormatter REPORT_DIR_FMT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH-mm-ss"); private static final String[] PYTHON_RESOURCES = { "mobile-ui-auto/python/runner.py", - "mobile-ui-auto/python/requirements.txt" + "mobile-ui-auto/python/requirements.txt", + "mobile-ui-auto/python/install_tools.py" }; private static final String[] TEMPLATE_RESOURCES = { "mobile-ui-auto/templates/sample-login-flow.yaml" @@ -41,6 +45,9 @@ public class MobileAutomationService { private final Map running = new ConcurrentHashMap<>(); private RecordingSession recordingSession; + + // 默认等待时间（毫秒），可通过 UI 配置修改 + private int defaultWaitTimeMs = 500; @PostConstruct public void init() { @@ -73,6 +80,67 @@ public class MobileAutomationService { return venvDir; } + /** + * 设置默认等待时间（毫秒） + */ + public void setDefaultWaitTime(int waitMs) { + this.defaultWaitTimeMs = Math.max(100, waitMs); // 最小 100ms + } + + /** + * 获取当前默认等待时间 + */ + public int getDefaultWaitTime() { + return defaultWaitTimeMs; + } + + /** + * 列出执行历史记录（从报告目录读取 result.json） + * @param limit 最大返回数量 + * @return 按时间倒序排列的历史记录 + */ + public List listExecutionHistory(int limit) { + List entries = new ArrayList<>(); + if (!Files.isDirectory(reportsDir)) { + return entries; + } + try (DirectoryStream stream = Files.newDirectoryStream(reportsDir)) { + for (Path dir : stream) { + if (!Files.isDirectory(dir)) continue; + Path resultFile = dir.resolve("result.json"); + if (!Files.exists(resultFile)) continue; + try { + String content = new String(Files.readAllBytes(resultFile), StandardCharsets.UTF_8); + JSONObject json = JSON.parseObject(content); + ExecutionHistoryEntry entry = new ExecutionHistoryEntry(); + entry.reportDir = dir; + entry.caseName = json.getString("caseName"); + entry.device = json.getString("device"); + entry.packageName = json.getString("package"); + entry.success = Boolean.TRUE.equals(json.getBoolean("success")); + entry.startTime = json.getString("startTime"); + entry.endTime = json.getString("endTime"); + entry.steps = json.getJSONArray("steps") != null ? json.getJSONArray("steps").toJavaList(Object.class) : Collections.emptyList(); + entries.add(entry); + } catch (Exception e) { + log.debug("解析报告失败: {} - {}", resultFile, e.getMessage()); + } + } + } catch (IOException e) { + log.warn("读取报告目录失败: {}", e.getMessage()); + } + // 按开始时间倒序排列 + entries.sort((a, b) -> { + String ta = a.startTime != null ? a.startTime : ""; + String tb = b.startTime != null ? b.startTime : ""; + return tb.compareTo(ta); + }); + if (limit > 0 && entries.size() > limit) { + return entries.subList(0, limit); + } + return entries; + } + public void initializeProjectEnvironment(AutomationCallback callback) throws IOException { Files.createDirectories(baseDir); Files.createDirectories(pythonDir); @@ -113,6 +181,56 @@ public class MobileAutomationService { callback.onEvent("log", "INFO", "Python 依赖安装完成"); } + /** + * 安装辅助截图工具（minicap/yosemite）到指定设备 + */ + public void installAssistTools(String deviceSerial, AutomationCallback callback) throws IOException { + Path python = resolvePythonExecutable(); + if (!Files.exists(python)) { + throw new IOException("Python 环境未初始化，请先点击\"初始化环境\""); + } + String deviceUri = "Android:///" + nullToEmpty(deviceSerial); + List command = Arrays.asList( + python.toString(), + pythonDir.resolve("install_tools.py").toString(), + "--device", deviceUri, + "--tool", "all", + "--marker-dir", baseDir.toString() + ); + callback.onEvent("log", "INFO", "正在安装辅助截图工具到设备: " + deviceSerial); + CmdResult result = runStreamingCommand(command, baseDir, 120, callback); + if (result.ok) { + callback.onEvent("log", "INFO", "辅助工具安装完成"); + } else { + callback.onEvent("log", "WARN", "部分工具安装失败，运行用例时将自动回退到 ADB 截图方式"); + } + } + + /** + * 检查指定设备是否已安装 minicap + */ + public boolean isMinicapInstalled(String deviceSerial) { + String safeSerial = sanitizeSerial(deviceSerial); + Path marker = baseDir.resolve(".minicap-" + safeSerial); + return Files.exists(marker); + } + + /** + * 检查指定设备是否已安装 Yosemite + */ + public boolean isYosemiteInstalled(String deviceSerial) { + String safeSerial = sanitizeSerial(deviceSerial); + Path marker = baseDir.resolve(".yosemite-" + safeSerial); + return Files.exists(marker); + } + + private String sanitizeSerial(String serial) { + if (serial == null || serial.trim().isEmpty()) { + return "default"; + } + return serial.trim().replace(":", "_").replace("/", "_"); + } + public EnvironmentCheckResult checkEnvironment() { EnvironmentCheckResult result = new EnvironmentCheckResult(); Path python = resolvePythonExecutable(); @@ -184,8 +302,11 @@ public class MobileAutomationService { public AutomationRunHandle runCase(AutomationRunConfig config, AutomationCallback callback) throws IOException { validateRunConfig(config); - String runId = LocalDateTime.now().format(RUN_ID_FMT); - Path reportDir = reportsDir.resolve(runId); + LocalDateTime now = LocalDateTime.now(); + String runId = now.format(RUN_ID_FMT); + // 报告目录使用可读格式: 报告 2026-07-06 19-13-02 + String reportDirName = "报告 " + now.format(REPORT_DIR_FMT); + Path reportDir = reportsDir.resolve(reportDirName); Files.createDirectories(reportDir); List command = new ArrayList<>(); @@ -206,9 +327,18 @@ public class MobileAutomationService { if (config.isScreenshotOnFail()) { command.add("--screenshot-on-fail"); } + // 如果设备已安装辅助工具，启用高效截图模式 + if (isMinicapInstalled(config.getDeviceSerial())) { + command.add("--use-minicap"); + callback.onEvent("log", "INFO", "minicap 已安装，启用高效截图模式"); + } else if (isYosemiteInstalled(config.getDeviceSerial())) { + command.add("--use-javacap"); + callback.onEvent("log", "INFO", "Yosemite 已安装，启用 javacap 高效截图模式"); + } ProcessBuilder pb = new ProcessBuilder(command); pb.directory(baseDir.toFile()); + pb.environment().put("PYTHONIOENCODING", "utf-8"); pb.redirectErrorStream(false); Process process = pb.start(); AutomationRunHandle handle = new AutomationRunHandle(runId, process); @@ -253,10 +383,86 @@ public class MobileAutomationService { } public synchronized void startRecording(String deviceSerial, String packageName) throws IOException { - String runId = "record-" + LocalDateTime.now().format(RUN_ID_FMT); - Path recordDir = reportsDir.resolve(runId); + LocalDateTime now = LocalDateTime.now(); + String runId = "record-" + now.format(RUN_ID_FMT); + // 报告目录使用可读格式: 报告 2026-07-06 19-13-02 + String reportDirName = "报告 " + now.format(REPORT_DIR_FMT); + Path recordDir = reportsDir.resolve(reportDirName); Files.createDirectories(recordDir.resolve("screenshots")); - recordingSession = new RecordingSession(runId, deviceSerial, packageName, recordDir); + + // 如果未提供包名，尝试从设备获取当前前台应用 + String actualPackageName = packageName; + if (actualPackageName == null || actualPackageName.trim().isEmpty()) { + actualPackageName = getCurrentForegroundApp(deviceSerial); + if (actualPackageName != null) { + log.info("自动检测到前台应用: {}", actualPackageName); + } + } + + recordingSession = new RecordingSession(runId, deviceSerial, actualPackageName, recordDir, now); + } + + /** + * 获取设备当前前台应用的包名（排除桌面启动器） + */ + public String getCurrentForegroundApp(String deviceSerial) { + try { + List command = new ArrayList<>(); + command.add(resolveAdbExecutable().orElse(Path.of("adb")).toString()); + if (deviceSerial != null && !deviceSerial.trim().isEmpty()) { + command.add("-s"); + command.add(deviceSerial.trim()); + } + // 使用 dumpsys window 获取当前焦点窗口 + command.addAll(Arrays.asList("shell", "dumpsys", "window", "|", "grep", "mCurrentFocus")); + + CmdResult result = runCommand(command, null, 5); + if (result.ok && result.output != null && !result.output.isEmpty()) { + // 输出格式类似: mCurrentFocus=Window{... u0 com.example.app/com.example.app.MainActivity} + String output = result.output.trim(); + java.util.regex.Pattern pattern = java.util.regex.Pattern.compile("u\\d+\\s+([a-zA-Z0-9._]+)"); + java.util.regex.Matcher matcher = pattern.matcher(output); + if (matcher.find()) { + String packageName = matcher.group(1); + // 排除桌面启动器包名 + if (isLauncherPackage(packageName)) { + log.info("检测到桌面启动器包名: {}，跳过自动填充", packageName); + return null; + } + return packageName; + } + } + } catch (Exception e) { + log.warn("获取前台应用失败: {}", e.getMessage()); + } + return null; + } + + /** + * 判断是否为桌面启动器包名 + */ + private static final Set LAUNCHER_PACKAGES = Set.of( + "com.miui.home", // 小米 + "com.android.launcher", // AOSP + "com.android.launcher3", // AOSP + "com.sec.android.app.launcher", // 三星 + "com.huawei.android.launcher", // 华为 + "com.oppo.launcher", // OPPO + "com.coloros.launcher", // OPPO ColorOS + "com.bbk.launcher", // Vivo + "com.vivo.launcher", // Vivo + "com.oneplus.launcher", // 一加 + "com.google.android.apps.nexuslauncher", // Pixel + "com.teslacoilsw.launcher", // Nova Launcher + "com.motorola.launcher3" // 摩托罗拉 + ); + + private boolean isLauncherPackage(String packageName) { + if (packageName == null) return false; + String lower = packageName.toLowerCase(Locale.ROOT); + if (LAUNCHER_PACKAGES.contains(lower)) return true; + // 通配匹配：包含 launcher 关键字的包名 + return lower.contains(".launcher") || lower.endsWith(".home"); } public synchronized boolean isRecording() { @@ -267,8 +473,13 @@ public class MobileAutomationService { if (recordingSession == null) { throw new IOException("尚未开始录制"); } - Path screenshot = recordingSession.recordDir.resolve("screenshots/current.png"); + // 使用带时间戳的文件名避免覆盖，同时保留 current.png 作为最新截图的快捷方式 + String timestamp = LocalDateTime.now().format(RUN_ID_FMT); + Path screenshot = recordingSession.recordDir.resolve("screenshots/screenshot-" + timestamp + ".png"); captureScreenshot(recordingSession.deviceSerial, screenshot); + // 同步创建 current.png 软链接（或复制），供实时显示使用 + Path current = recordingSession.recordDir.resolve("screenshots/current.png"); + Files.copy(screenshot, current, StandardCopyOption.REPLACE_EXISTING); return screenshot; } @@ -279,7 +490,7 @@ public class MobileAutomationService { int x = scale(imageX, imageWidth, screenWidth); int y = scale(imageY, imageHeight, screenHeight); runAdb(recordingSession.deviceSerial, Arrays.asList("shell", "input", "tap", String.valueOf(x), String.valueOf(y)), 10); - recordingSession.addTap(x, y, screenWidth, screenHeight); + recordingSession.addTap(x, y, screenWidth, screenHeight, defaultWaitTimeMs); } public synchronized void recordInput(String text) throws IOException { @@ -290,7 +501,7 @@ public class MobileAutomationService { text = ""; } runAdb(recordingSession.deviceSerial, Arrays.asList("shell", "input", "text", escapeAdbInput(text)), 10); - recordingSession.addInput(text); + recordingSession.addInput(text, defaultWaitTimeMs); } public synchronized void recordBack() throws IOException { @@ -298,7 +509,7 @@ public class MobileAutomationService { throw new IOException("尚未开始录制"); } runAdb(recordingSession.deviceSerial, Arrays.asList("shell", "input", "keyevent", "BACK"), 10); - recordingSession.addKeyevent("BACK"); + recordingSession.addKeyevent("BACK", defaultWaitTimeMs); } public synchronized void recordSwipe(int startX, int startY, int endX, int endY, int durationMs) throws IOException { @@ -307,22 +518,362 @@ public class MobileAutomationService { } runAdb(recordingSession.deviceSerial, Arrays.asList("shell", "input", "swipe", String.valueOf(startX), String.valueOf(startY), String.valueOf(endX), String.valueOf(endY), String.valueOf(durationMs)), 10); - recordingSession.addSwipe(startX, startY, endX, endY, durationMs); + recordingSession.addSwipe(startX, startY, endX, endY, durationMs, defaultWaitTimeMs); + } + + /** + * 撤销上一步录制操作 + * @return 被撤销的步骤描述，null 表示没有可撤销的步骤 + */ + public synchronized String undoLastStep() { + if (recordingSession == null || recordingSession.steps.isEmpty()) { + return null; + } + JSONObject removed = recordingSession.steps.remove(recordingSession.steps.size() - 1); + recordingSession.index--; + // 返回步骤名称用于显示 + return removed.getString("name"); + } + + /** + * 更新指定步骤的等待时间 + * @param index 步骤索引（从0开始） + * @param waitMs 新的等待时间（毫秒） + * @return 是否成功 + */ + public synchronized boolean updateStepWaitTime(int index, int waitMs) { + if (recordingSession == null || index < 0 || index >= recordingSession.steps.size()) { + return false; + } + JSONObject step = recordingSession.steps.get(index); + step.put("waitAfterMs", waitMs); + return true; + } + + /** + * 删除指定索引的步骤 + * @param index 步骤索引（从0开始） + * @return 是否成功 + */ + public synchronized boolean deleteStep(int index) { + if (recordingSession == null || index < 0 || index >= recordingSession.steps.size()) { + return false; + } + recordingSession.steps.remove(index); + // 重新编号后续步骤 + for (int i = index; i < recordingSession.steps.size(); i++) { + JSONObject step = recordingSession.steps.get(i); + String name = step.getString("name"); + if (name != null) { + // 将 tap-N、input-N 等改为正确的序号 + String newName = name.replaceAll("-(\\d+)$", "-" + (i + 1)); + step.put("name", newName); + } + } + recordingSession.index = recordingSession.steps.size() + 1; + return true; + } + + /** + * 设置指定步骤的页面特征 + * @param index 步骤索引（从0开始） + * @param featureName 页面特征名称（如"login-page"、"ad-popup"等） + * @param screenshotPath 截图路径（可选，null表示不设置截图） + * @return 是否成功 + */ + public synchronized boolean setStepPageFeature(int index, String featureName, String screenshotPath) { + if (recordingSession == null || index < 0 || index >= recordingSession.steps.size()) { + return false; + } + JSONObject step = recordingSession.steps.get(index); + if (featureName == null || featureName.trim().isEmpty()) { + step.remove("pageFeature"); + step.remove("pageFeatureScreenshot"); + } else { + step.put("pageFeature", featureName.trim()); + if (screenshotPath != null && !screenshotPath.isEmpty()) { + step.put("pageFeatureScreenshot", screenshotPath); + } + } + return true; + } + + /** + * 获取指定步骤的页面特征信息 + * @param index 步骤索引（从0开始） + * @return 页面特征名称，如果没有则返回 null + */ + public synchronized String getStepPageFeature(int index) { + if (recordingSession == null || index < 0 || index >= recordingSession.steps.size()) { + return null; + } + JSONObject step = recordingSession.steps.get(index); + return step.getString("pageFeature"); + } + + /** + * 获取指定步骤的完整JSON信息 + * @param index 步骤索引（从0开始） + * @return 步骤的 JSONObject，如果无效则返回 null + */ + public synchronized JSONObject getStepInfo(int index) { + if (recordingSession == null || index < 0 || index >= recordingSession.steps.size()) { + return null; + } + return recordingSession.steps.get(index); + } + + /** + * 获取设备当前页面的 UI 层级信息 + * @param deviceSerial 设备序列号 + * @return UI 元素列表，每个元素包含 resourceId, text, contentDesc, className, bounds + */ + public List dumpUiHierarchy(String deviceSerial) throws IOException { + // 1. 在设备上生成 UI dump + Path localDump = baseDir.resolve("window_dump.xml"); + runAdb(deviceSerial, Arrays.asList("shell", "uiautomator", "dump", "/sdcard/window_dump.xml"), 15); + // 2. 拉取到本地 + runAdb(deviceSerial, Arrays.asList("pull", "/sdcard/window_dump.xml", localDump.toString()), 10); + // 3. 解析 XML + List elements = new ArrayList<>(); + if (!Files.exists(localDump)) { + throw new IOException("UI dump 文件不存在"); + } + try { + String xmlContent = new String(Files.readAllBytes(localDump), StandardCharsets.UTF_8); + // 简单解析 XML 中的 node 元素 + javax.xml.parsers.DocumentBuilderFactory factory = javax.xml.parsers.DocumentBuilderFactory.newInstance(); + javax.xml.parsers.DocumentBuilder builder = factory.newDocumentBuilder(); + org.w3c.dom.Document doc = builder.parse(new ByteArrayInputStream(xmlContent.getBytes(StandardCharsets.UTF_8))); + org.w3c.dom.NodeList nodes = doc.getElementsByTagName("node"); + for (int i = 0; i < nodes.getLength(); i++) { + org.w3c.dom.Element node = (org.w3c.dom.Element) nodes.item(i); + JSONObject elem = new JSONObject(); + elem.put("resourceId", node.getAttribute("resource-id")); + elem.put("text", node.getAttribute("text")); + elem.put("contentDesc", node.getAttribute("content-desc")); + elem.put("className", node.getAttribute("class")); + elem.put("bounds", node.getAttribute("bounds")); + elem.put("clickable", "true".equals(node.getAttribute("clickable"))); + elem.put("enabled", "true".equals(node.getAttribute("enabled"))); + // 解析 bounds: [left,top][right,bottom] + String bounds = node.getAttribute("bounds"); + if (bounds != null && !bounds.isEmpty()) { + java.util.regex.Matcher m = java.util.regex.Pattern.compile("\\[(\\d+),(\\d+)\\]\\[(\\d+),(\\d+)\\]").matcher(bounds); + if (m.find()) { + elem.put("left", Integer.parseInt(m.group(1))); + elem.put("top", Integer.parseInt(m.group(2))); + elem.put("right", Integer.parseInt(m.group(3))); + elem.put("bottom", Integer.parseInt(m.group(4))); + } + } + elements.add(elem); + } + } catch (javax.xml.parsers.ParserConfigurationException | org.xml.sax.SAXException e) { + throw new IOException("解析 UI dump XML 失败: " + e.getMessage()); + } finally { + try { Files.deleteIfExists(localDump); } catch (IOException ignored) {} + } + return elements; + } + + /** + * 识别指定坐标处的 UI 元素 + * @param deviceSerial 设备序列号 + * @param x 屏幕 X 坐标 + * @param y 屏幕 Y 坐标 + * @return 匹配的元素列表（按优先级排序：resource-id > content-desc > text） + */ + public List identifyElement(String deviceSerial, int x, int y) throws IOException { + List allElements = dumpUiHierarchy(deviceSerial); + List matched = new ArrayList<>(); + for (JSONObject elem : allElements) { + Integer left = elem.getInteger("left"); + Integer top = elem.getInteger("top"); + Integer right = elem.getInteger("right"); + Integer bottom = elem.getInteger("bottom"); + if (left != null && top != null && right != null && bottom != null) { + if (x >= left && x <= right && y >= top && y <= bottom) { + // 计算面积用于排序（优先选择更精确的小元素） + int area = (right - left) * (bottom - top); + elem.put("area", area); + matched.add(elem); + } + } + } + // 按面积升序排序（面积越小越精确） + matched.sort((a, b) -> a.getIntValue("area") - b.getIntValue("area")); + return matched; + } + + /** + * 更新指定步骤的目标元素信息（元素优先+坐标兑底） + * @param index 步骤索引 + * @param element 识别到的元素信息（来自 identifyElement） + * @return 是否成功 + */ + public synchronized boolean updateStepElement(int index, JSONObject element) { + if (recordingSession == null || index < 0 || index >= recordingSession.steps.size()) { + return false; + } + JSONObject step = recordingSession.steps.get(index); + JSONObject target = step.getJSONObject("target"); + if (target == null) { + target = new JSONObject(); + step.put("target", target); + } + // 设置 Poco 查询信息 + JSONObject poco = new JSONObject(); + String resourceId = element.getString("resourceId"); + String contentDesc = element.getString("contentDesc"); + String text = element.getString("text"); + if (resourceId != null && !resourceId.isEmpty()) { + poco.put("name", resourceId); + } else if (contentDesc != null && !contentDesc.isEmpty()) { + poco.put("name", contentDesc); + } else if (text != null && !text.isEmpty()) { + poco.put("text", text); + } else { + return false; // 无可识别属性 + } + target.put("preferred", "poco"); + target.put("poco", poco); + return true; + } + + /** + * 更新已保存用例文件中的步骤元素信息 + * @param caseFile 用例文件路径 + * @param stepIndex 步骤索引 + * @param element 识别到的元素信息 + * @return 是否成功 + */ + public boolean updateCaseStepElement(Path caseFile, int stepIndex, JSONObject element) throws IOException { + if (caseFile == null || !Files.exists(caseFile)) { + return false; + } + String content = new String(Files.readAllBytes(caseFile), StandardCharsets.UTF_8); + JSONObject caseJson = JSON.parseObject(content); + com.alibaba.fastjson.JSONArray steps = caseJson.getJSONArray("steps"); + if (steps == null || stepIndex < 0 || stepIndex >= steps.size()) { + return false; + } + JSONObject step = steps.getJSONObject(stepIndex); + JSONObject target = step.getJSONObject("target"); + if (target == null) { + target = new JSONObject(); + step.put("target", target); + } + JSONObject poco = new JSONObject(); + String resourceId = element.getString("resourceId"); + String contentDesc = element.getString("contentDesc"); + String text = element.getString("text"); + if (resourceId != null && !resourceId.isEmpty()) { + poco.put("name", resourceId); + } else if (contentDesc != null && !contentDesc.isEmpty()) { + poco.put("name", contentDesc); + } else if (text != null && !text.isEmpty()) { + poco.put("text", text); + } else { + return false; + } + target.put("preferred", "poco"); + target.put("poco", poco); + Files.write(caseFile, caseJson.toJSONString().getBytes(StandardCharsets.UTF_8)); + return true; + } + + /** + * 对已保存的用例文件进行 UI dump 并识别指定步骤的元素 + * @param deviceSerial 设备序列号 + * @param caseFile 用例文件 + * @param stepIndex 步骤索引 + * @return 匹配的元素列表 + */ + public List identifyCaseStepElement(String deviceSerial, Path caseFile, int stepIndex) throws IOException { + if (caseFile == null || !Files.exists(caseFile)) { + throw new IOException("用例文件不存在"); + } + String content = new String(Files.readAllBytes(caseFile), StandardCharsets.UTF_8); + JSONObject caseJson = JSON.parseObject(content); + com.alibaba.fastjson.JSONArray steps = caseJson.getJSONArray("steps"); + if (steps == null || stepIndex < 0 || stepIndex >= steps.size()) { + throw new IOException("步骤索引无效"); + } + JSONObject step = steps.getJSONObject(stepIndex); + JSONObject target = step.getJSONObject("target"); + if (target == null || !"tap".equals(step.getString("action"))) { + return Collections.emptyList(); + } + int x = target.getIntValue("x"); + int y = target.getIntValue("y"); + return identifyElement(deviceSerial, x, y); } public synchronized Path stopRecording() throws IOException { if (recordingSession == null) { throw new IOException("尚未开始录制"); } - Path file = casesDir.resolve(recordingSession.runId + ".yaml"); - Files.write(file, recordingSession.toYaml().getBytes(StandardCharsets.UTF_8)); - Path latest = casesDir.resolve("latest-recorded-flow.yaml"); + // 生成可读用例文件名: 2026-07-06 19-13 用例（N步）.json + int stepCount = recordingSession.steps.size(); + String caseName = recordingSession.startTime.format(CASE_NAME_FMT) + " 用例（" + stepCount + "步）"; + Path file = casesDir.resolve(caseName + ".json"); + // 更新 JSON 内的 name 字段 + recordingSession.caseName = caseName; + Files.write(file, recordingSession.toJSON().getBytes(StandardCharsets.UTF_8)); + Path latest = casesDir.resolve("latest-recorded-flow.json"); Files.copy(file, latest, StandardCopyOption.REPLACE_EXISTING); recordingSession = null; return file; } - private void captureScreenshot(String serial, Path output) throws IOException { + /** + * 重命名用例文件 + */ + public Path renameCase(Path oldFile, String newName) throws IOException { + if (oldFile == null || !Files.exists(oldFile)) { + throw new IOException("文件不存在"); + } + // 清理文件名中的非法字符 + String safeName = newName.replaceAll("[\\\\/:*?\"<>|]", "-"); + Path newFile = oldFile.getParent().resolve(safeName + ".json"); + if (Files.exists(newFile) && !newFile.equals(oldFile)) { + throw new IOException("文件名已存在: " + safeName); + } + // 更新 JSON 内部的 name 字段 + try { + String content = new String(Files.readAllBytes(oldFile), StandardCharsets.UTF_8); + JSONObject json = JSON.parseObject(content); + json.put("name", safeName); + Files.write(oldFile, json.toJSONString().getBytes(StandardCharsets.UTF_8)); + } catch (Exception e) { + log.warn("更新用例name字段失败: {}", e.getMessage()); + } + if (!newFile.equals(oldFile)) { + Files.move(oldFile, newFile, StandardCopyOption.REPLACE_EXISTING); + } + return newFile; + } + + /** + * 重命名报告目录 + */ + public Path renameReport(Path oldDir, String newName) throws IOException { + if (oldDir == null || !Files.isDirectory(oldDir)) { + throw new IOException("报告目录不存在"); + } + String safeName = newName.replaceAll("[\\\\/:*?\"<>|]", "-"); + Path newDir = oldDir.getParent().resolve(safeName); + if (Files.exists(newDir) && !newDir.equals(oldDir)) { + throw new IOException("目录名已存在: " + safeName); + } + if (!newDir.equals(oldDir)) { + Files.move(oldDir, newDir, StandardCopyOption.REPLACE_EXISTING); + } + return newDir; + } + + public void captureScreenshot(String serial, Path output) throws IOException { List command = new ArrayList<>(); command.add(resolveAdbExecutable().orElse(Path.of("adb")).toString()); if (serial != null && !serial.trim().isEmpty()) { @@ -364,7 +915,7 @@ public class MobileAutomationService { private boolean isCaseFile(Path path) { String name = path.getFileName().toString().toLowerCase(Locale.ROOT); - return name.endsWith(".yaml") || name.endsWith(".yml") || name.endsWith(".py") || name.endsWith(".air"); + return name.endsWith(".json") || name.endsWith(".yaml") || name.endsWith(".yml") || name.endsWith(".py") || name.endsWith(".air"); } private void addCheck(EnvironmentCheckResult result, String name, List command) { @@ -388,6 +939,19 @@ public class MobileAutomationService { try (BufferedReader reader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8))) { String line; while ((line = reader.readLine()) != null) { + // 过滤 Airtest 内部 DEBUG 日志（Airtest 将 DEBUG 日志写到 stderr） + if (line.contains("[DEBUG]") || line.contains("[DEBUG]<")) { + continue; // 跳过 DEBUG 级别日志 + } + // 过滤 Airtest 内部的 INFO 日志（也写到 stderr） + if (line.matches(".*\\[INFO\\] adb = resolveAdbExecutable(); + if (adb.isEmpty()) return false; + CmdResult result = runCommand(Arrays.asList(adb.get().toString(), "-s", serial, "get-state"), null, 5); + return result.ok && result.output.trim().equals("device"); + } catch (Exception e) { + return false; + } + } + private void runAdb(String serial, List args, int timeoutSeconds) throws IOException { CmdResult result = runCommand(buildAdbCommand(serial, args), null, timeoutSeconds); if (!result.ok) { @@ -476,6 +1054,8 @@ public class MobileAutomationService { if (workingDir != null) { pb.directory(workingDir.toFile()); } + // 确保 Python 输出使用 UTF-8 编码，避免中文乱码 + pb.environment().put("PYTHONIOENCODING", "utf-8"); pb.redirectErrorStream(true); process = pb.start(); Process proc = process; @@ -679,73 +1259,113 @@ public class MobileAutomationService { final String deviceSerial; final String packageName; final Path recordDir; - final List steps = new ArrayList<>(); + final LocalDateTime startTime; + String caseName; // 停止录制时设置 + final List steps = new ArrayList<>(); int index = 1; - RecordingSession(String runId, String deviceSerial, String packageName, Path recordDir) { + RecordingSession(String runId, String deviceSerial, String packageName, Path recordDir, LocalDateTime startTime) { this.runId = runId; this.deviceSerial = deviceSerial; this.packageName = packageName; this.recordDir = recordDir; + this.startTime = startTime; + this.caseName = runId; // 默认值 } - void addTap(int x, int y, int screenWidth, int screenHeight) { - steps.add(" - name: tap-" + index + "\n" - + " action: tap\n" - + " target:\n" - + " type: coord\n" - + " preferred: coord\n" - + " x: " + x + "\n" - + " y: " + y + "\n" - + " screenWidth: " + screenWidth + "\n" - + " screenHeight: " + screenHeight + "\n" - + " waitAfterMs: 500"); + void addTap(int x, int y, int screenWidth, int screenHeight, int waitAfterMs) { + JSONObject step = new JSONObject(); + step.put("name", "tap-" + index); + step.put("action", "tap"); + + JSONObject target = new JSONObject(); + target.put("type", "coord"); + target.put("preferred", "coord"); + target.put("x", x); + target.put("y", y); + target.put("screenWidth", screenWidth); + target.put("screenHeight", screenHeight); + step.put("target", target); + step.put("waitAfterMs", waitAfterMs); + // 可选：页面特征（用于执行时验证当前页面） + // step.put("pageFeature", "login-page"); + + steps.add(step); index++; } - void addInput(String text) { - steps.add(" - name: input-" + index + "\n" - + " action: input_text\n" - + " text: \"" + yamlEscape(text) + "\"\n" - + " waitAfterMs: 500"); + void addInput(String text, int waitAfterMs) { + JSONObject step = new JSONObject(); + step.put("name", "input-" + index); + step.put("action", "input_text"); + step.put("text", text); + step.put("waitAfterMs", waitAfterMs); + + steps.add(step); index++; } - void addKeyevent(String key) { - steps.add(" - name: key-" + index + "\n" - + " action: keyevent\n" - + " key: " + key + "\n" - + " waitAfterMs: 500"); + void addKeyevent(String key, int waitAfterMs) { + JSONObject step = new JSONObject(); + step.put("name", "key-" + index); + step.put("action", "keyevent"); + step.put("key", key); + step.put("waitAfterMs", waitAfterMs); + + steps.add(step); index++; } - void addSwipe(int startX, int startY, int endX, int endY, int durationMs) { - steps.add(" - name: swipe-" + index + "\n" - + " action: swipe\n" - + " start: {x: " + startX + ", y: " + startY + "}\n" - + " end: {x: " + endX + ", y: " + endY + "}\n" - + " durationMs: " + durationMs + "\n" - + " waitAfterMs: 500"); + void addSwipe(int startX, int startY, int endX, int endY, int durationMs, int waitAfterMs) { + JSONObject step = new JSONObject(); + step.put("name", "swipe-" + index); + step.put("action", "swipe"); + + JSONObject start = new JSONObject(); + start.put("x", startX); + start.put("y", startY); + step.put("start", start); + + JSONObject end = new JSONObject(); + end.put("x", endX); + end.put("y", endY); + step.put("end", end); + + step.put("durationMs", durationMs); + step.put("waitAfterMs", waitAfterMs); + + steps.add(step); index++; } - String toYaml() { - StringBuilder yaml = new StringBuilder(); - yaml.append("name: ").append(runId).append("\n"); - yaml.append("app:\n"); - yaml.append(" package: ").append(packageName == null ? "" : packageName).append("\n"); - yaml.append(" activity: \n"); - yaml.append("device:\n"); - yaml.append(" connect: Android:///"); - yaml.append(deviceSerial == null ? "" : deviceSerial).append("\n"); - yaml.append("settings:\n"); - yaml.append(" timeout: 20\n"); - yaml.append(" screenshotOnFail: true\n"); - yaml.append("steps:\n"); - for (String step : steps) { - yaml.append(step).append("\n"); - } - return yaml.toString(); + /** + * 生成 JSON 格式的用例文件 + */ + String toJSON() { + JSONObject caseJson = new JSONObject(); + caseJson.put("name", caseName); + + // App 信息 + JSONObject app = new JSONObject(); + app.put("package", packageName == null ? "" : packageName); + app.put("activity", ""); + caseJson.put("app", app); + + // 设备信息 + JSONObject device = new JSONObject(); + device.put("connect", "Android:///" + (deviceSerial == null ? "" : deviceSerial)); + caseJson.put("device", device); + + // 设置 + JSONObject settings = new JSONObject(); + settings.put("timeout", 20); + settings.put("screenshotOnFail", true); + caseJson.put("settings", settings); + + // 步骤列表 + caseJson.put("steps", steps); + + return caseJson.toJSONString(); } private String yamlEscape(String text) { diff --git a/src/main/java/com/example/workutils/biz/mobileuiauto/service/model/ExecutionHistoryEntry.java b/src/main/java/com/example/workutils/biz/mobileuiauto/service/model/ExecutionHistoryEntry.java new file mode 100644 index 0000000..3877e4c --- /dev/null +++ b/src/main/java/com/example/workutils/biz/mobileuiauto/service/model/ExecutionHistoryEntry.java @@ -0,0 +1,36 @@ +package com.example.workutils.biz.mobileuiauto.service.model; + +import java.nio.file.Path; +import java.util.List; + +/** + * 执行历史记录条目 + */ +public class ExecutionHistoryEntry { + /** 报告目录 */ + public Path reportDir; + /** 用例名称 */ + public String caseName; + /** 设备标识 */ + public String device; + /** 包名 */ + public String packageName; + /** 是否成功 */ + public boolean success; + /** 开始时间（ISO 格式字符串） */ + public String startTime; + /** 结束时间（ISO 格式字符串） */ + public String endTime; + /** 步骤列表 */ + public List steps; + + @Override + public String toString() { + String icon = success ? "✅" : "❌"; + String time = startTime != null && startTime.length() >= 16 + ? startTime.substring(0, 16).replace("T", " ") + : startTime; + int stepCount = steps != null ? steps.size() : 0; + return icon + " " + caseName + " (" + stepCount + "步) " + time; + } +} diff --git a/src/main/resources/css/light-theme.css b/src/main/resources/css/light-theme.css index ce4a71b..0e09501 100644 --- a/src/main/resources/css/light-theme.css +++ b/src/main/resources/css/light-theme.css @@ -291,3 +291,18 @@ -fx-padding: 8 0; -fx-alignment: CENTER_LEFT; } + +/* ---- 紧凑列表单元格 ---- */ +.compact-list .list-cell { + -fx-padding: 3 8 3 8; + -fx-font-size: 13; +} + +.compact-list .list-cell:selected { + -fx-background-color: derive(-workutils-primary, 85%); + -fx-text-fill: -workutils-text; +} + +.compact-list .list-cell:filled:hover { + -fx-background-color: -workutils-card; +} diff --git a/src/main/resources/mobile-ui-auto/python/install_tools.py b/src/main/resources/mobile-ui-auto/python/install_tools.py new file mode 100644 index 0000000..f5b867b --- /dev/null +++ b/src/main/resources/mobile-ui-auto/python/install_tools.py @@ -0,0 +1,286 @@ +"""安装 Airtest 辅助工具（minicap/yosemite）到设备""" +import argparse +import json +import os +import subprocess +import sys +import time + + +def emit(event_type, level="INFO", message="", **extra): + payload = {"type": event_type, "level": level, "message": message} + payload.update(extra) + print(json.dumps(payload, ensure_ascii=False), flush=True) + + +def find_adb(): + """查找 adb 可执行文件""" + venv_dir = os.path.join(os.path.dirname(__file__), "..", ".venv") + candidates = [ + os.path.join(venv_dir, "Lib", "site-packages", "airtest", "core", "android", "static", "adb", "windows", "adb.exe"), + os.path.join(venv_dir, "lib", "python3.11", "site-packages", "airtest", "core", "android", "static", "adb", "linux", "adb"), + os.path.join(venv_dir, "lib", "python3.10", "site-packages", "airtest", "core", "android", "static", "adb", "linux", "adb"), + ] + for c in candidates: + if os.path.exists(c): + return c + return "adb" + + +def install_minicap(device_uri): + """通过 Airtest 安装 minicap 并验证""" + try: + from airtest.core.api import auto_setup + import logging + logging.getLogger("airtest").setLevel(logging.INFO) + + # 提取 serial + serial = device_uri.replace("Android:///", "").strip() + serial = serial.split("?")[0] if "?" in serial else serial + + emit("log", "INFO", "正在连接设备并安装 minicap...") + # 不使用 cap_method=adbcap，让 Airtest 尝试安装 minicap + auto_setup(__file__, devices=[device_uri]) + + # 等待安装完成 + time.sleep(3) + + # 通过 ADB 检查 minicap 二进制文件是否存在于设备上 + adb = find_adb() + check_cmd = [adb] + if serial: + check_cmd.extend(["-s", serial]) + check_cmd.extend(["shell", "ls", "/data/local/tmp/minicap"]) + + emit("log", "INFO", "检查 minicap 是否已安装到设备...") + result = subprocess.run(check_cmd, capture_output=True, text=True, timeout=10) + combined = (result.stdout + result.stderr).strip() + + if "No such file" in combined or "not found" in combined.lower(): + emit("log", "WARN", "minicap 二进制文件未找到，安装失败") + return False + + # minicap 文件存在，进一步验证是否能正常运行 + # 尝试启动 minicap 并立即停止 + test_cmd = [adb] + if serial: + test_cmd.extend(["-s", serial]) + test_cmd.extend(["shell", "LD_LIBRARY_PATH=/data/local/tmp", "/data/local/tmp/minicap", "-i"]) + + emit("log", "INFO", "验证 minicap 是否可正常运行...") + try: + test_result = subprocess.run(test_cmd, capture_output=True, text=True, timeout=5) + test_output = (test_result.stdout + test_result.stderr).strip() + if "pid=" in test_output or "web socket" in test_output.lower() or result.returncode == 0: + emit("log", "INFO", "minicap 安装成功且可正常运行") + return True + else: + emit("log", "WARN", f"minicap 运行测试输出: {test_output}") + # 即使测试输出不明确，文件存在也算安装成功 + return True + except subprocess.TimeoutExpired: + # minicap 作为服务运行会超时，这是正常的 + emit("log", "INFO", "minicap 安装成功（服务进程已启动）") + return True + + except Exception as e: + emit("log", "ERROR", f"安装 minicap 失败: {e}") + return False + + +def install_yosemite(device_uri): + """安装 Yosemite APK（用于 javacap 截图）""" + try: + from airtest.core.api import auto_setup, connect_device + import logging + logging.getLogger("airtest").setLevel(logging.INFO) + + serial = device_uri.replace("Android:///", "").strip() + serial = serial.split("?")[0] if "?" in serial else serial + + emit("log", "INFO", "正在连接设备...") + auto_setup(__file__, devices=[device_uri]) + time.sleep(1) + + # 通过 Airtest 的 Android 设备对象直接安装 Yosemite + from airtest.core.helper import G + device = G.DEVICE + + emit("log", "INFO", "正在安装 Yosemite APK...") + # 使用设备对象的 yosemite 安装方法 + try: + from airtest.core.android.yosemite import Yosemite + yosemite = Yosemite(device.adb) + yosemite.install_or_upgrade() + emit("log", "INFO", "Yosemite 安装成功") + return True + except TypeError: + # 旧版本 API 兼容 + from airtest.core.android.yosemite import Yosemite + yosemite = Yosemite(device) + yosemite.install_or_upgrade() + emit("log", "INFO", "Yosemite 安装成功") + return True + except Exception as e: + emit("log", "WARN", f"Yosemite 安装失败: {e}") + return False + + +def main(): + parser = argparse.ArgumentParser(description="安装 Airtest 辅助工具") + parser.add_argument("--device", default="Android:///") + parser.add_argument("--tool", default="minicap", choices=["minicap", "yosemite", "all"]) + parser.add_argument("--marker-dir", required=True, help="标记文件存放目录") + args = parser.parse_args() + + os.makedirs(args.marker_dir, exist_ok=True) + # 从设备 URI 中提取 serial 用于标记文件名 + serial = args.device.replace("Android:///", "").strip() + serial = serial.split("?")[0] if "?" in serial else serial + serial = serial if serial else "default" + safe_serial = serial.replace(":", "_").replace("/", "_") + + results = {"minicap": False, "yosemite": False} + + if args.tool in ("minicap", "all"): + results["minicap"] = install_minicap(args.device) + + if args.tool in ("yosemite", "all"): + results["yosemite"] = install_yosemite(args.device) + + # 写入标记文件 + for tool_name, success in results.items(): + marker_file = os.path.join(args.marker_dir, f".{tool_name}-{safe_serial}") + if success: + with open(marker_file, "w") as f: + f.write(json.dumps({"installed": True, "time": time.strftime("%Y-%m-%d %H:%M:%S")})) + emit("log", "INFO", f"{tool_name} 安装成功，已写入标记文件") + else: + # 如果失败，删除已有标记文件 + if os.path.exists(marker_file): + os.remove(marker_file) + emit("log", "WARN", f"{tool_name} 安装失败，已清除标记") + + # 输出最终结果 + any_success = any(results.values()) + emit("result", "INFO" if any_success else "ERROR", + json.dumps(results, ensure_ascii=False), + success=any_success) + return 0 if any_success else 1 + + +if __name__ == "__main__": + sys.exit(main()) +"""安装 Airtest 辅助工具（minicap/yosemite）到设备""" +import argparse +import json +import os +import sys +import time + + +def emit(event_type, level="INFO", message="", **extra): + payload = {"type": event_type, "level": level, "message": message} + payload.update(extra) + print(json.dumps(payload, ensure_ascii=False), flush=True) + + +def install_minicap(device_uri): + """通过 Airtest 安装 minicap 并验证""" + try: + from airtest.core.api import auto_setup, snapshot + from airtest.core.android import Android as AndroidDevice + import logging + logging.getLogger("airtest").setLevel(logging.INFO) + + emit("log", "INFO", "正在连接设备...") + # 不使用 cap_method=adbcap，让 Airtest 尝试安装 minicap + auto_setup(__file__, devices=[device_uri]) + + emit("log", "INFO", "设备已连接，正在初始化截图组件...") + # Airtest 在 auto_setup 时会自动安装 minicap， + # 尝试截图来验证是否安装成功 + time.sleep(2) + test_path = os.path.join(os.path.dirname(__file__), "..", "test_minicap.png") + try: + snapshot(filename=test_path) + if os.path.exists(test_path) and os.path.getsize(test_path) > 100: + os.remove(test_path) + emit("log", "INFO", "minicap 安装成功，截图验证通过") + return True + else: + emit("log", "WARN", "截图文件过小，minicap 可能未正确安装") + if os.path.exists(test_path): + os.remove(test_path) + return False + except Exception as e: + emit("log", "WARN", f"minicap 截图测试失败: {e}") + return False + except Exception as e: + emit("log", "ERROR", f"安装 minicap 失败: {e}") + return False + + +def install_yosemite(device_uri): + """安装 Yosemite APK（用于 javacap 截图）""" + try: + from airtest.core.api import auto_setup + from airtest.core.android.yosemite import Yosemite + import logging + logging.getLogger("airtest").setLevel(logging.INFO) + + emit("log", "INFO", "正在安装 Yosemite APK...") + auto_setup(__file__, devices=[device_uri]) + yosemite = Yosemite() + yosemite.install_or_upgrade() + emit("log", "INFO", "Yosemite 安装成功") + return True + except Exception as e: + emit("log", "WARN", f"Yosemite 安装失败: {e}") + return False + + +def main(): + parser = argparse.ArgumentParser(description="安装 Airtest 辅助工具") + parser.add_argument("--device", default="Android:///") + parser.add_argument("--tool", default="minicap", choices=["minicap", "yosemite", "all"]) + parser.add_argument("--marker-dir", required=True, help="标记文件存放目录") + args = parser.parse_args() + + os.makedirs(args.marker_dir, exist_ok=True) + # 从设备 URI 中提取 serial 用于标记文件名 + serial = args.device.replace("Android:///", "").strip() + serial = serial.split("?")[0] if "?" in serial else serial + serial = serial if serial else "default" + safe_serial = serial.replace(":", "_").replace("/", "_") + + results = {"minicap": False, "yosemite": False} + + if args.tool in ("minicap", "all"): + results["minicap"] = install_minicap(args.device) + + if args.tool in ("yosemite", "all"): + results["yosemite"] = install_yosemite(args.device) + + # 写入标记文件 + for tool_name, success in results.items(): + marker_file = os.path.join(args.marker_dir, f".{tool_name}-{safe_serial}") + if success: + with open(marker_file, "w") as f: + f.write(json.dumps({"installed": True, "time": time.strftime("%Y-%m-%d %H:%M:%S")})) + emit("log", "INFO", f"{tool_name} 安装成功，已写入标记文件") + else: + # 如果失败，删除已有标记文件 + if os.path.exists(marker_file): + os.remove(marker_file) + + # 输出最终结果 + any_success = any(results.values()) + emit("result", "INFO" if any_success else "ERROR", + json.dumps(results, ensure_ascii=False), + success=any_success) + return 0 if any_success else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/main/resources/mobile-ui-auto/python/runner.py b/src/main/resources/mobile-ui-auto/python/runner.py index 165cf29..5b51902 100644 --- a/src/main/resources/mobile-ui-auto/python/runner.py +++ b/src/main/resources/mobile-ui-auto/python/runner.py @@ -14,13 +14,26 @@ def emit(event_type, level="INFO", message="", **extra): def load_yaml(path): + """加载 YAML 或 JSON 格式的用例文件""" + import json try: import yaml + has_yaml = True except Exception as exc: + has_yaml = False + + # 尝试 JSON 格式（优先） + if path.endswith('.json'): + with open(path, "r", encoding="utf-8") as fh: + return json.load(fh) or {} + + # 尝试 YAML 格式 + if has_yaml: + with open(path, "r", encoding="utf-8") as fh: + return yaml.safe_load(fh) or {} + else: emit("log", "ERROR", "缺少 PyYAML，请执行: python -m pip install -U pyyaml") - raise exc - with open(path, "r", encoding="utf-8") as fh: - return yaml.safe_load(fh) or {} + raise ImportError("PyYAML not installed") def ensure_airtest(): @@ -80,13 +93,31 @@ def run_air_case(case_path, args): def connect(api, args): os.makedirs(args.report, exist_ok=True) - api["auto_setup"](__file__, logdir=args.report, devices=[args.device]) + # 抑制 Airtest 内部日志输出到 stderr（避免 DEBUG 日志污染输出） + import logging + logging.getLogger("airtest").setLevel(logging.WARNING) + logging.getLogger("poco").setLevel(logging.WARNING) + # 构造带参数的设备 URI + device_uri = args.device + use_minicap = getattr(args, "use_minicap", False) + use_javacap = getattr(args, "use_javacap", False) + if device_uri.startswith("Android:///") and "cap_method" not in device_uri: + sep = "?" if "?" not in device_uri else "&" + if use_minicap: + # minicap 已安装，使用默认截图方式（Airtest 自动选择 minicap） + device_uri = device_uri + sep + "touch_method=maxtouch" + emit("log", "INFO", "使用 minicap 高效截图模式") + elif use_javacap: + # Yosemite 已安装，使用 javacap 截图 + device_uri = device_uri + sep + "cap_method=javacap&touch_method=maxtouch" + emit("log", "INFO", "使用 javacap (Yosemite) 高效截图模式") + else: + # 未安装辅助工具，强制使用 adbcap 避免安装尝试 + device_uri = device_uri + sep + "cap_method=adbcap&touch_method=maxtouch" + emit("log", "INFO", "使用 ADB 截图模式（可通过\"安装辅助工具\"启用高效截图）") + api["auto_setup"](__file__, logdir=args.report, devices=[device_uri]) poco = None - try: - poco = api["AndroidUiautomationPoco"](use_airtest_input=True, screenshot_each_action=False) - emit("log", "INFO", "Poco 初始化成功") - except Exception as exc: - emit("log", "WARN", "Poco 初始化失败，将仅使用 Airtest/坐标/图片定位: " + str(exc)) + emit("log", "INFO", "纯坐标模式运行，跳过 Poco 初始化") return poco @@ -161,6 +192,48 @@ def assert_exists(api, poco, target, case_dir): raise AssertionError("图片不存在: " + target["path"]) +def verify_page_feature(api, step, case_dir): + """验证当前页面特征，返回 (matched, message)""" + page_feature = step.get("pageFeature") + if not page_feature: + return True, None + + screenshot_path = step.get("pageFeatureScreenshot") + if screenshot_path: + # 有特征截图，尝试比对 + try: + import cv2 + import numpy as np + # 获取当前截图 + current_screenshot = os.path.join(case_dir, f"current_verify_{int(time.time()*1000)}.png") + api["snapshot"](filename=current_screenshot) + # 加载特征截图 + feature_img = cv2.imread(os.path.join(case_dir, screenshot_path)) + current_img = cv2.imread(current_screenshot) + if feature_img is not None and current_img is not None: + # 调整大小一致 + h, w = feature_img.shape[:2] + current_img = cv2.resize(current_img, (w, h)) + # 计算结构相似性 + diff = cv2.absdiff(feature_img, current_img) + mean_diff = np.mean(diff) + similarity = 1 - (mean_diff / 255.0) + os.remove(current_screenshot) + if similarity > 0.85: + return True, f"页面特征 '{page_feature}' 匹配 (相似度: {similarity:.2f})" + else: + return False, f"页面特征 '{page_feature}' 不匹配 (相似度: {similarity:.2f})" + os.remove(current_screenshot) + except ImportError: + pass + except Exception as e: + emit("log", "WARN", f"页面特征比对失败: {e}") + + # 无特征截图或比对失败，仅记录日志 + emit("log", "INFO", f"页面特征: {page_feature}") + return True, f"页面特征 '{page_feature}' 已记录（无截图比对）" + + def run_yaml_case(case_path, args): api = ensure_airtest() case = load_yaml(case_path) @@ -180,12 +253,29 @@ def run_yaml_case(case_path, args): "steps": [], } + # 常见桌面启动器包名，start_app 对启动器会失败 + LAUNCHER_PACKAGES = { + "com.miui.home", "com.android.launcher", "com.android.launcher2", + "com.android.launcher3", "com.sec.android.app.launcher", + "com.huawei.android.launcher", "com.oppo.launcher", + "com.coloros.oppolauncher", "com.vivo.launcher", + "com.oneplus.launcher", "com.google.android.apps.nexuslauncher", + "com.google.android.launcher", "com.teslacoilsw.launcher", + "com.microsoft.launcher", + } + if package_name: - emit("log", "INFO", "启动 APP: " + package_name) - if activity_name: - api["start_app"](package_name, activity=activity_name) + if package_name.lower() in LAUNCHER_PACKAGES: + emit("log", "WARN", f"{package_name} 是桌面启动器，跳过启动（请手动打开被测应用）") else: - api["start_app"](package_name) + emit("log", "INFO", "启动 APP: " + package_name) + try: + if activity_name: + api["start_app"](package_name, activity=activity_name) + else: + api["start_app"](package_name) + except Exception as e: + emit("log", "WARN", f"启动应用失败: {e}，继续执行用例") for idx, step in enumerate(steps, start=1): name = step.get("name") or f"step-{idx}" @@ -194,6 +284,15 @@ def run_yaml_case(case_path, args): emit("step_start", "INFO", f"{idx}. {name}", index=idx, name=name) record = {"index": idx, "name": name, "action": action, "status": "PASS"} try: + # 页面特征验证（如果步骤定义了 pageFeature） + matched, feature_msg = verify_page_feature(api, step, case_dir) + if not matched: + emit("log", "WARN", f"步骤 {idx} 页面特征不匹配: {feature_msg}") + # 页面特征不匹配时，可以选择跳过或继续执行 + # 这里选择继续执行，但记录警告 + elif feature_msg: + emit("log", "INFO", feature_msg) + if action in ("tap", "click"): tap_target(api, poco, step.get("target"), case_dir) elif action in ("input", "input_text"): @@ -269,6 +368,8 @@ def main(): parser.add_argument("--report", required=True) parser.add_argument("--timeout", type=int, default=20) parser.add_argument("--screenshot-on-fail", action="store_true") + parser.add_argument("--use-minicap", action="store_true", help="启用 minicap 高效截图（需已安装辅助工具）") + parser.add_argument("--use-javacap", action="store_true", help="启用 javacap 高效截图（需已安装 Yosemite）") args = parser.parse_args() case_path = os.path.abspath(args.case)