UI自动化能力优化

src/main/resources/css/light-theme.css

@@ -291,3 +291,18 @@
     -fx-padding: 8 0;
     -fx-alignment: CENTER_LEFT;
 }
+
+/* ---- 紧凑列表单元格 ---- */
+.compact-list .list-cell {
+    -fx-padding: 3 8 3 8;
+    -fx-font-size: 13;
+}
+
+.compact-list .list-cell:selected {
+    -fx-background-color: derive(-workutils-primary, 85%);
+    -fx-text-fill: -workutils-text;
+}
+
+.compact-list .list-cell:filled:hover {
+    -fx-background-color: -workutils-card;
+}

src/main/resources/mobile-ui-auto/python/install_tools.py

@@ -0,0 +1,286 @@
+"""安装 Airtest 辅助工具（minicap/yosemite）到设备"""
+import argparse
+import json
+import os
+import subprocess
+import sys
+import time
+
+
+def emit(event_type, level="INFO", message="", **extra):
+    payload = {"type": event_type, "level": level, "message": message}
+    payload.update(extra)
+    print(json.dumps(payload, ensure_ascii=False), flush=True)
+
+
+def find_adb():
+    """查找 adb 可执行文件"""
+    venv_dir = os.path.join(os.path.dirname(__file__), "..", ".venv")
+    candidates = [
+        os.path.join(venv_dir, "Lib", "site-packages", "airtest", "core", "android", "static", "adb", "windows", "adb.exe"),
+        os.path.join(venv_dir, "lib", "python3.11", "site-packages", "airtest", "core", "android", "static", "adb", "linux", "adb"),
+        os.path.join(venv_dir, "lib", "python3.10", "site-packages", "airtest", "core", "android", "static", "adb", "linux", "adb"),
+    ]
+    for c in candidates:
+        if os.path.exists(c):
+            return c
+    return "adb"
+
+
+def install_minicap(device_uri):
+    """通过 Airtest 安装 minicap 并验证"""
+    try:
+        from airtest.core.api import auto_setup
+        import logging
+        logging.getLogger("airtest").setLevel(logging.INFO)
+
+        # 提取 serial
+        serial = device_uri.replace("Android:///", "").strip()
+        serial = serial.split("?")[0] if "?" in serial else serial
+
+        emit("log", "INFO", "正在连接设备并安装 minicap...")
+        # 不使用 cap_method=adbcap，让 Airtest 尝试安装 minicap
+        auto_setup(__file__, devices=[device_uri])
+
+        # 等待安装完成
+        time.sleep(3)
+
+        # 通过 ADB 检查 minicap 二进制文件是否存在于设备上
+        adb = find_adb()
+        check_cmd = [adb]
+        if serial:
+            check_cmd.extend(["-s", serial])
+        check_cmd.extend(["shell", "ls", "/data/local/tmp/minicap"])
+
+        emit("log", "INFO", "检查 minicap 是否已安装到设备...")
+        result = subprocess.run(check_cmd, capture_output=True, text=True, timeout=10)
+        combined = (result.stdout + result.stderr).strip()
+
+        if "No such file" in combined or "not found" in combined.lower():
+            emit("log", "WARN", "minicap 二进制文件未找到，安装失败")
+            return False
+
+        # minicap 文件存在，进一步验证是否能正常运行
+        # 尝试启动 minicap 并立即停止
+        test_cmd = [adb]
+        if serial:
+            test_cmd.extend(["-s", serial])
+        test_cmd.extend(["shell", "LD_LIBRARY_PATH=/data/local/tmp", "/data/local/tmp/minicap", "-i"])
+
+        emit("log", "INFO", "验证 minicap 是否可正常运行...")
+        try:
+            test_result = subprocess.run(test_cmd, capture_output=True, text=True, timeout=5)
+            test_output = (test_result.stdout + test_result.stderr).strip()
+            if "pid=" in test_output or "web socket" in test_output.lower() or result.returncode == 0:
+                emit("log", "INFO", "minicap 安装成功且可正常运行")
+                return True
+            else:
+                emit("log", "WARN", f"minicap 运行测试输出: {test_output}")
+                # 即使测试输出不明确，文件存在也算安装成功
+                return True
+        except subprocess.TimeoutExpired:
+            # minicap 作为服务运行会超时，这是正常的
+            emit("log", "INFO", "minicap 安装成功（服务进程已启动）")
+            return True
+
+    except Exception as e:
+        emit("log", "ERROR", f"安装 minicap 失败: {e}")
+        return False
+
+
+def install_yosemite(device_uri):
+    """安装 Yosemite APK（用于 javacap 截图）"""
+    try:
+        from airtest.core.api import auto_setup, connect_device
+        import logging
+        logging.getLogger("airtest").setLevel(logging.INFO)
+
+        serial = device_uri.replace("Android:///", "").strip()
+        serial = serial.split("?")[0] if "?" in serial else serial
+
+        emit("log", "INFO", "正在连接设备...")
+        auto_setup(__file__, devices=[device_uri])
+        time.sleep(1)
+
+        # 通过 Airtest 的 Android 设备对象直接安装 Yosemite
+        from airtest.core.helper import G
+        device = G.DEVICE
+
+        emit("log", "INFO", "正在安装 Yosemite APK...")
+        # 使用设备对象的 yosemite 安装方法
+        try:
+            from airtest.core.android.yosemite import Yosemite
+            yosemite = Yosemite(device.adb)
+            yosemite.install_or_upgrade()
+            emit("log", "INFO", "Yosemite 安装成功")
+            return True
+        except TypeError:
+            # 旧版本 API 兼容
+            from airtest.core.android.yosemite import Yosemite
+            yosemite = Yosemite(device)
+            yosemite.install_or_upgrade()
+            emit("log", "INFO", "Yosemite 安装成功")
+            return True
+    except Exception as e:
+        emit("log", "WARN", f"Yosemite 安装失败: {e}")
+        return False
+
+
+def main():
+    parser = argparse.ArgumentParser(description="安装 Airtest 辅助工具")
+    parser.add_argument("--device", default="Android:///")
+    parser.add_argument("--tool", default="minicap", choices=["minicap", "yosemite", "all"])
+    parser.add_argument("--marker-dir", required=True, help="标记文件存放目录")
+    args = parser.parse_args()
+
+    os.makedirs(args.marker_dir, exist_ok=True)
+    # 从设备 URI 中提取 serial 用于标记文件名
+    serial = args.device.replace("Android:///", "").strip()
+    serial = serial.split("?")[0] if "?" in serial else serial
+    serial = serial if serial else "default"
+    safe_serial = serial.replace(":", "_").replace("/", "_")
+
+    results = {"minicap": False, "yosemite": False}
+
+    if args.tool in ("minicap", "all"):
+        results["minicap"] = install_minicap(args.device)
+
+    if args.tool in ("yosemite", "all"):
+        results["yosemite"] = install_yosemite(args.device)
+
+    # 写入标记文件
+    for tool_name, success in results.items():
+        marker_file = os.path.join(args.marker_dir, f".{tool_name}-{safe_serial}")
+        if success:
+            with open(marker_file, "w") as f:
+                f.write(json.dumps({"installed": True, "time": time.strftime("%Y-%m-%d %H:%M:%S")}))
+            emit("log", "INFO", f"{tool_name} 安装成功，已写入标记文件")
+        else:
+            # 如果失败，删除已有标记文件
+            if os.path.exists(marker_file):
+                os.remove(marker_file)
+            emit("log", "WARN", f"{tool_name} 安装失败，已清除标记")
+
+    # 输出最终结果
+    any_success = any(results.values())
+    emit("result", "INFO" if any_success else "ERROR",
+         json.dumps(results, ensure_ascii=False),
+         success=any_success)
+    return 0 if any_success else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+"""安装 Airtest 辅助工具（minicap/yosemite）到设备"""
+import argparse
+import json
+import os
+import sys
+import time
+
+
+def emit(event_type, level="INFO", message="", **extra):
+    payload = {"type": event_type, "level": level, "message": message}
+    payload.update(extra)
+    print(json.dumps(payload, ensure_ascii=False), flush=True)
+
+
+def install_minicap(device_uri):
+    """通过 Airtest 安装 minicap 并验证"""
+    try:
+        from airtest.core.api import auto_setup, snapshot
+        from airtest.core.android import Android as AndroidDevice
+        import logging
+        logging.getLogger("airtest").setLevel(logging.INFO)
+
+        emit("log", "INFO", "正在连接设备...")
+        # 不使用 cap_method=adbcap，让 Airtest 尝试安装 minicap
+        auto_setup(__file__, devices=[device_uri])
+
+        emit("log", "INFO", "设备已连接，正在初始化截图组件...")
+        # Airtest 在 auto_setup 时会自动安装 minicap，
+        # 尝试截图来验证是否安装成功
+        time.sleep(2)
+        test_path = os.path.join(os.path.dirname(__file__), "..", "test_minicap.png")
+        try:
+            snapshot(filename=test_path)
+            if os.path.exists(test_path) and os.path.getsize(test_path) > 100:
+                os.remove(test_path)
+                emit("log", "INFO", "minicap 安装成功，截图验证通过")
+                return True
+            else:
+                emit("log", "WARN", "截图文件过小，minicap 可能未正确安装")
+                if os.path.exists(test_path):
+                    os.remove(test_path)
+                return False
+        except Exception as e:
+            emit("log", "WARN", f"minicap 截图测试失败: {e}")
+            return False
+    except Exception as e:
+        emit("log", "ERROR", f"安装 minicap 失败: {e}")
+        return False
+
+
+def install_yosemite(device_uri):
+    """安装 Yosemite APK（用于 javacap 截图）"""
+    try:
+        from airtest.core.api import auto_setup
+        from airtest.core.android.yosemite import Yosemite
+        import logging
+        logging.getLogger("airtest").setLevel(logging.INFO)
+
+        emit("log", "INFO", "正在安装 Yosemite APK...")
+        auto_setup(__file__, devices=[device_uri])
+        yosemite = Yosemite()
+        yosemite.install_or_upgrade()
+        emit("log", "INFO", "Yosemite 安装成功")
+        return True
+    except Exception as e:
+        emit("log", "WARN", f"Yosemite 安装失败: {e}")
+        return False
+
+
+def main():
+    parser = argparse.ArgumentParser(description="安装 Airtest 辅助工具")
+    parser.add_argument("--device", default="Android:///")
+    parser.add_argument("--tool", default="minicap", choices=["minicap", "yosemite", "all"])
+    parser.add_argument("--marker-dir", required=True, help="标记文件存放目录")
+    args = parser.parse_args()
+
+    os.makedirs(args.marker_dir, exist_ok=True)
+    # 从设备 URI 中提取 serial 用于标记文件名
+    serial = args.device.replace("Android:///", "").strip()
+    serial = serial.split("?")[0] if "?" in serial else serial
+    serial = serial if serial else "default"
+    safe_serial = serial.replace(":", "_").replace("/", "_")
+
+    results = {"minicap": False, "yosemite": False}
+
+    if args.tool in ("minicap", "all"):
+        results["minicap"] = install_minicap(args.device)
+
+    if args.tool in ("yosemite", "all"):
+        results["yosemite"] = install_yosemite(args.device)
+
+    # 写入标记文件
+    for tool_name, success in results.items():
+        marker_file = os.path.join(args.marker_dir, f".{tool_name}-{safe_serial}")
+        if success:
+            with open(marker_file, "w") as f:
+                f.write(json.dumps({"installed": True, "time": time.strftime("%Y-%m-%d %H:%M:%S")}))
+            emit("log", "INFO", f"{tool_name} 安装成功，已写入标记文件")
+        else:
+            # 如果失败，删除已有标记文件
+            if os.path.exists(marker_file):
+                os.remove(marker_file)
+
+    # 输出最终结果
+    any_success = any(results.values())
+    emit("result", "INFO" if any_success else "ERROR",
+         json.dumps(results, ensure_ascii=False),
+         success=any_success)
+    return 0 if any_success else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/main/resources/mobile-ui-auto/python/runner.py

@@ -14,13 +14,26 @@ def emit(event_type, level="INFO", message="", **extra):
 
 
 def load_yaml(path):
+    """加载 YAML 或 JSON 格式的用例文件"""
+    import json
     try:
         import yaml
+        has_yaml = True
     except Exception as exc:
+        has_yaml = False
+    
+    # 尝试 JSON 格式（优先）
+    if path.endswith('.json'):
+        with open(path, "r", encoding="utf-8") as fh:
+            return json.load(fh) or {}
+    
+    # 尝试 YAML 格式
+    if has_yaml:
+        with open(path, "r", encoding="utf-8") as fh:
+            return yaml.safe_load(fh) or {}
+    else:
         emit("log", "ERROR", "缺少 PyYAML，请执行: python -m pip install -U pyyaml")
-        raise exc
-    with open(path, "r", encoding="utf-8") as fh:
-        return yaml.safe_load(fh) or {}
+        raise ImportError("PyYAML not installed")
 
 
 def ensure_airtest():
@@ -80,13 +93,31 @@ def run_air_case(case_path, args):
 
 def connect(api, args):
     os.makedirs(args.report, exist_ok=True)
-    api["auto_setup"](__file__, logdir=args.report, devices=[args.device])
+    # 抑制 Airtest 内部日志输出到 stderr（避免 DEBUG 日志污染输出）
+    import logging
+    logging.getLogger("airtest").setLevel(logging.WARNING)
+    logging.getLogger("poco").setLevel(logging.WARNING)
+    # 构造带参数的设备 URI
+    device_uri = args.device
+    use_minicap = getattr(args, "use_minicap", False)
+    use_javacap = getattr(args, "use_javacap", False)
+    if device_uri.startswith("Android:///") and "cap_method" not in device_uri:
+        sep = "?" if "?" not in device_uri else "&"
+        if use_minicap:
+            # minicap 已安装，使用默认截图方式（Airtest 自动选择 minicap）
+            device_uri = device_uri + sep + "touch_method=maxtouch"
+            emit("log", "INFO", "使用 minicap 高效截图模式")
+        elif use_javacap:
+            # Yosemite 已安装，使用 javacap 截图
+            device_uri = device_uri + sep + "cap_method=javacap&touch_method=maxtouch"
+            emit("log", "INFO", "使用 javacap (Yosemite) 高效截图模式")
+        else:
+            # 未安装辅助工具，强制使用 adbcap 避免安装尝试
+            device_uri = device_uri + sep + "cap_method=adbcap&touch_method=maxtouch"
+            emit("log", "INFO", "使用 ADB 截图模式（可通过\"安装辅助工具\"启用高效截图）")
+    api["auto_setup"](__file__, logdir=args.report, devices=[device_uri])
     poco = None
-    try:
-        poco = api["AndroidUiautomationPoco"](use_airtest_input=True, screenshot_each_action=False)
-        emit("log", "INFO", "Poco 初始化成功")
-    except Exception as exc:
-        emit("log", "WARN", "Poco 初始化失败，将仅使用 Airtest/坐标/图片定位: " + str(exc))
+    emit("log", "INFO", "纯坐标模式运行，跳过 Poco 初始化")
     return poco
 
 
@@ -161,6 +192,48 @@ def assert_exists(api, poco, target, case_dir):
             raise AssertionError("图片不存在: " + target["path"])
 
 
+def verify_page_feature(api, step, case_dir):
+    """验证当前页面特征，返回 (matched, message)"""
+    page_feature = step.get("pageFeature")
+    if not page_feature:
+        return True, None
+    
+    screenshot_path = step.get("pageFeatureScreenshot")
+    if screenshot_path:
+        # 有特征截图，尝试比对
+        try:
+            import cv2
+            import numpy as np
+            # 获取当前截图
+            current_screenshot = os.path.join(case_dir, f"current_verify_{int(time.time()*1000)}.png")
+            api["snapshot"](filename=current_screenshot)
+            # 加载特征截图
+            feature_img = cv2.imread(os.path.join(case_dir, screenshot_path))
+            current_img = cv2.imread(current_screenshot)
+            if feature_img is not None and current_img is not None:
+                # 调整大小一致
+                h, w = feature_img.shape[:2]
+                current_img = cv2.resize(current_img, (w, h))
+                # 计算结构相似性
+                diff = cv2.absdiff(feature_img, current_img)
+                mean_diff = np.mean(diff)
+                similarity = 1 - (mean_diff / 255.0)
+                os.remove(current_screenshot)
+                if similarity > 0.85:
+                    return True, f"页面特征 '{page_feature}' 匹配 (相似度: {similarity:.2f})"
+                else:
+                    return False, f"页面特征 '{page_feature}' 不匹配 (相似度: {similarity:.2f})"
+            os.remove(current_screenshot)
+        except ImportError:
+            pass
+        except Exception as e:
+            emit("log", "WARN", f"页面特征比对失败: {e}")
+    
+    # 无特征截图或比对失败，仅记录日志
+    emit("log", "INFO", f"页面特征: {page_feature}")
+    return True, f"页面特征 '{page_feature}' 已记录（无截图比对）"
+
+
 def run_yaml_case(case_path, args):
     api = ensure_airtest()
     case = load_yaml(case_path)
@@ -180,12 +253,29 @@ def run_yaml_case(case_path, args):
         "steps": [],
     }
 
+    # 常见桌面启动器包名，start_app 对启动器会失败
+    LAUNCHER_PACKAGES = {
+        "com.miui.home", "com.android.launcher", "com.android.launcher2",
+        "com.android.launcher3", "com.sec.android.app.launcher",
+        "com.huawei.android.launcher", "com.oppo.launcher",
+        "com.coloros.oppolauncher", "com.vivo.launcher",
+        "com.oneplus.launcher", "com.google.android.apps.nexuslauncher",
+        "com.google.android.launcher", "com.teslacoilsw.launcher",
+        "com.microsoft.launcher",
+    }
+
     if package_name:
-        emit("log", "INFO", "启动 APP: " + package_name)
-        if activity_name:
-            api["start_app"](package_name, activity=activity_name)
+        if package_name.lower() in LAUNCHER_PACKAGES:
+            emit("log", "WARN", f"{package_name} 是桌面启动器，跳过启动（请手动打开被测应用）")
         else:
-            api["start_app"](package_name)
+            emit("log", "INFO", "启动 APP: " + package_name)
+            try:
+                if activity_name:
+                    api["start_app"](package_name, activity=activity_name)
+                else:
+                    api["start_app"](package_name)
+            except Exception as e:
+                emit("log", "WARN", f"启动应用失败: {e}，继续执行用例")
 
     for idx, step in enumerate(steps, start=1):
         name = step.get("name") or f"step-{idx}"
@@ -194,6 +284,15 @@ def run_yaml_case(case_path, args):
         emit("step_start", "INFO", f"{idx}. {name}", index=idx, name=name)
         record = {"index": idx, "name": name, "action": action, "status": "PASS"}
         try:
+            # 页面特征验证（如果步骤定义了 pageFeature）
+            matched, feature_msg = verify_page_feature(api, step, case_dir)
+            if not matched:
+                emit("log", "WARN", f"步骤 {idx} 页面特征不匹配: {feature_msg}")
+                # 页面特征不匹配时，可以选择跳过或继续执行
+                # 这里选择继续执行，但记录警告
+            elif feature_msg:
+                emit("log", "INFO", feature_msg)
+            
             if action in ("tap", "click"):
                 tap_target(api, poco, step.get("target"), case_dir)
             elif action in ("input", "input_text"):
@@ -269,6 +368,8 @@ def main():
     parser.add_argument("--report", required=True)
     parser.add_argument("--timeout", type=int, default=20)
     parser.add_argument("--screenshot-on-fail", action="store_true")
+    parser.add_argument("--use-minicap", action="store_true", help="启用 minicap 高效截图（需已安装辅助工具）")
+    parser.add_argument("--use-javacap", action="store_true", help="启用 javacap 高效截图（需已安装 Yosemite）")
     args = parser.parse_args()
 
     case_path = os.path.abspath(args.case)