From 633ecff4e2d2fff8568f38fd14c8a3d50a085590 Mon Sep 17 00:00:00 2001 From: chendelian <116870791@qq.com> Date: Fri, 7 Aug 2026 11:19:14 +0800 Subject: [PATCH] chore: archive tools/ui_inspector snapshot before migration removal. Keep the last in-template copy on main so history can restore it; product continues in jiangchang-ui-recorder. Co-authored-by: Cursor --- tools/ui_inspector/.gitignore | 2 + tools/ui_inspector/README.md | 188 +- tools/ui_inspector/__init__.py | 4 +- tools/ui_inspector/adapters/__init__.py | 7 + tools/ui_inspector/adapters/base.py | 17 + tools/ui_inspector/adapters/registry.py | 31 + tools/ui_inspector/adapters/wecom.py | 103 + tools/ui_inspector/app.py | 1605 ++++++++++++++-- tools/ui_inspector/artifacts.py | 61 +- tools/ui_inspector/compatibility/__init__.py | 2 + tools/ui_inspector/compatibility/v1_reader.py | 6 + tools/ui_inspector/compatibility/v1_writer.py | 6 + tools/ui_inspector/core/__init__.py | 2 + tools/ui_inspector/core/models/__init__.py | 16 + tools/ui_inspector/core/pipeline/__init__.py | 6 + .../core/pipeline/event_journal.py | 49 + tools/ui_inspector/core/policies/__init__.py | 7 + .../core/policies/element_identity.py | 62 + .../core/policies/target_scoring.py | 145 ++ tools/ui_inspector/desktop_uia.py | 220 ++- tools/ui_inspector/diagnostics/__init__.py | 6 + .../diagnostics/capture_validator.py | 134 ++ .../diagnostics/replay_preflight.py | 133 ++ tools/ui_inspector/drivers/__init__.py | 29 + tools/ui_inspector/drivers/base.py | 38 + tools/ui_inspector/drivers/browser_pw.py | 90 + tools/ui_inspector/drivers/mobile.py | 30 + tools/ui_inspector/drivers/windows_uia.py | 58 + tools/ui_inspector/evidence/__init__.py | 6 + tools/ui_inspector/evidence/coordinator.py | 255 +++ tools/ui_inspector/flow_session.py | 408 ++++ tools/ui_inspector/kit_browser.py | 729 +++++++ tools/ui_inspector/kit_wake.py | 110 +- tools/ui_inspector/locators/__init__.py | 13 + tools/ui_inspector/locators/base.py | 74 + tools/ui_inspector/locators/browser.py | 190 ++ tools/ui_inspector/locators/catalog.py | 294 +++ tools/ui_inspector/locators/validator.py | 50 + tools/ui_inspector/locators/windows.py | 180 ++ tools/ui_inspector/models/__init__.py | 22 + tools/ui_inspector/models/action.py | 119 ++ tools/ui_inspector/models/element.py | 126 ++ tools/ui_inspector/models/flow.py | 11 + tools/ui_inspector/models/locator.py | 40 + tools/ui_inspector/models/observation.py | 47 + tools/ui_inspector/models/serialize.py | 64 + tools/ui_inspector/models/state.py | 175 ++ tools/ui_inspector/recorder/__init__.py | 13 + tools/ui_inspector/recorder/base.py | 79 + tools/ui_inspector/recorder/browser.py | 740 +++++++ .../ui_inspector/recorder/input_aggregator.py | 234 +++ tools/ui_inspector/recorder/manager.py | 1220 ++++++++++++ tools/ui_inspector/recorder/windows.py | 1711 +++++++++++++++++ tools/ui_inspector/redaction.py | 206 ++ tools/ui_inspector/requirements.txt | 7 +- tools/ui_inspector/skip_policy.py | 89 + tools/ui_inspector/smoke_check.py | 255 ++- tools/ui_inspector/snapshot_service.py | 400 ++++ tools/ui_inspector/state_diff.py | 275 +++ tools/ui_inspector/storage/__init__.py | 6 + tools/ui_inspector/storage/capture_reader.py | 20 + tools/ui_inspector/storage/capture_writer.py | 6 + tools/ui_inspector/surfaces/__init__.py | 2 + tools/ui_inspector/surfaces/base.py | 19 + .../ui_inspector/surfaces/browser/__init__.py | 6 + .../surfaces/browser/dump_driver.py | 6 + .../surfaces/browser/locator_compiler.py | 6 + .../surfaces/browser/locator_validator.py | 6 + .../surfaces/browser/page_registry.py | 6 + .../ui_inspector/surfaces/browser/recorder.py | 6 + .../ui_inspector/surfaces/browser/runtime.py | 23 + .../ui_inspector/surfaces/mobile/__init__.py | 6 + tools/ui_inspector/surfaces/registry.py | 22 + .../ui_inspector/surfaces/windows/__init__.py | 6 + .../surfaces/windows/dump_driver.py | 4 + .../surfaces/windows/locator_compiler.py | 4 + .../ui_inspector/surfaces/windows/recorder.py | 4 + tools/ui_inspector/target.py | 82 + tools/ui_inspector/tests/__init__.py | 0 tools/ui_inspector/tests/contract/__init__.py | 1 + .../tests/contract/test_capture_contract.py | 207 ++ .../tests/test_locators_catalog.py | 215 +++ .../tests/test_models_fingerprint_diff.py | 295 +++ .../tests/test_recorder_adapters.py | 1513 +++++++++++++++ tools/ui_inspector/tests/unit/__init__.py | 1 + .../ui_inspector/tests/unit/core/__init__.py | 1 + .../unit/core/test_event_journal_identity.py | 30 + .../tests/unit/core/test_target_scoring.py | 70 + .../tests/unit/test_app_layout.py | 120 ++ tools/ui_inspector/ui/__init__.py | 2 + tools/ui_inspector/ui/controller.py | 10 + tools/ui_inspector/user_prefs.py | 88 + 92 files changed, 13704 insertions(+), 288 deletions(-) create mode 100644 tools/ui_inspector/adapters/__init__.py create mode 100644 tools/ui_inspector/adapters/base.py create mode 100644 tools/ui_inspector/adapters/registry.py create mode 100644 tools/ui_inspector/adapters/wecom.py create mode 100644 tools/ui_inspector/compatibility/__init__.py create mode 100644 tools/ui_inspector/compatibility/v1_reader.py create mode 100644 tools/ui_inspector/compatibility/v1_writer.py create mode 100644 tools/ui_inspector/core/__init__.py create mode 100644 tools/ui_inspector/core/models/__init__.py create mode 100644 tools/ui_inspector/core/pipeline/__init__.py create mode 100644 tools/ui_inspector/core/pipeline/event_journal.py create mode 100644 tools/ui_inspector/core/policies/__init__.py create mode 100644 tools/ui_inspector/core/policies/element_identity.py create mode 100644 tools/ui_inspector/core/policies/target_scoring.py create mode 100644 tools/ui_inspector/diagnostics/__init__.py create mode 100644 tools/ui_inspector/diagnostics/capture_validator.py create mode 100644 tools/ui_inspector/diagnostics/replay_preflight.py create mode 100644 tools/ui_inspector/drivers/__init__.py create mode 100644 tools/ui_inspector/drivers/base.py create mode 100644 tools/ui_inspector/drivers/browser_pw.py create mode 100644 tools/ui_inspector/drivers/mobile.py create mode 100644 tools/ui_inspector/drivers/windows_uia.py create mode 100644 tools/ui_inspector/evidence/__init__.py create mode 100644 tools/ui_inspector/evidence/coordinator.py create mode 100644 tools/ui_inspector/flow_session.py create mode 100644 tools/ui_inspector/kit_browser.py create mode 100644 tools/ui_inspector/locators/__init__.py create mode 100644 tools/ui_inspector/locators/base.py create mode 100644 tools/ui_inspector/locators/browser.py create mode 100644 tools/ui_inspector/locators/catalog.py create mode 100644 tools/ui_inspector/locators/validator.py create mode 100644 tools/ui_inspector/locators/windows.py create mode 100644 tools/ui_inspector/models/__init__.py create mode 100644 tools/ui_inspector/models/action.py create mode 100644 tools/ui_inspector/models/element.py create mode 100644 tools/ui_inspector/models/flow.py create mode 100644 tools/ui_inspector/models/locator.py create mode 100644 tools/ui_inspector/models/observation.py create mode 100644 tools/ui_inspector/models/serialize.py create mode 100644 tools/ui_inspector/models/state.py create mode 100644 tools/ui_inspector/recorder/__init__.py create mode 100644 tools/ui_inspector/recorder/base.py create mode 100644 tools/ui_inspector/recorder/browser.py create mode 100644 tools/ui_inspector/recorder/input_aggregator.py create mode 100644 tools/ui_inspector/recorder/manager.py create mode 100644 tools/ui_inspector/recorder/windows.py create mode 100644 tools/ui_inspector/redaction.py create mode 100644 tools/ui_inspector/skip_policy.py create mode 100644 tools/ui_inspector/snapshot_service.py create mode 100644 tools/ui_inspector/state_diff.py create mode 100644 tools/ui_inspector/storage/__init__.py create mode 100644 tools/ui_inspector/storage/capture_reader.py create mode 100644 tools/ui_inspector/storage/capture_writer.py create mode 100644 tools/ui_inspector/surfaces/__init__.py create mode 100644 tools/ui_inspector/surfaces/base.py create mode 100644 tools/ui_inspector/surfaces/browser/__init__.py create mode 100644 tools/ui_inspector/surfaces/browser/dump_driver.py create mode 100644 tools/ui_inspector/surfaces/browser/locator_compiler.py create mode 100644 tools/ui_inspector/surfaces/browser/locator_validator.py create mode 100644 tools/ui_inspector/surfaces/browser/page_registry.py create mode 100644 tools/ui_inspector/surfaces/browser/recorder.py create mode 100644 tools/ui_inspector/surfaces/browser/runtime.py create mode 100644 tools/ui_inspector/surfaces/mobile/__init__.py create mode 100644 tools/ui_inspector/surfaces/registry.py create mode 100644 tools/ui_inspector/surfaces/windows/__init__.py create mode 100644 tools/ui_inspector/surfaces/windows/dump_driver.py create mode 100644 tools/ui_inspector/surfaces/windows/locator_compiler.py create mode 100644 tools/ui_inspector/surfaces/windows/recorder.py create mode 100644 tools/ui_inspector/target.py create mode 100644 tools/ui_inspector/tests/__init__.py create mode 100644 tools/ui_inspector/tests/contract/__init__.py create mode 100644 tools/ui_inspector/tests/contract/test_capture_contract.py create mode 100644 tools/ui_inspector/tests/test_locators_catalog.py create mode 100644 tools/ui_inspector/tests/test_models_fingerprint_diff.py create mode 100644 tools/ui_inspector/tests/test_recorder_adapters.py create mode 100644 tools/ui_inspector/tests/unit/__init__.py create mode 100644 tools/ui_inspector/tests/unit/core/__init__.py create mode 100644 tools/ui_inspector/tests/unit/core/test_event_journal_identity.py create mode 100644 tools/ui_inspector/tests/unit/core/test_target_scoring.py create mode 100644 tools/ui_inspector/tests/unit/test_app_layout.py create mode 100644 tools/ui_inspector/ui/__init__.py create mode 100644 tools/ui_inspector/ui/controller.py create mode 100644 tools/ui_inspector/user_prefs.py diff --git a/tools/ui_inspector/.gitignore b/tools/ui_inspector/.gitignore index 0e4970c..1bbae20 100644 --- a/tools/ui_inspector/.gitignore +++ b/tools/ui_inspector/.gitignore @@ -1,5 +1,7 @@ # 仅本工具目录使用；不修改 skill-template 根 .gitignore _captures/ +_browser_profile/ +user_prefs.json __pycache__/ *.pyc .pytest_cache/ diff --git a/tools/ui_inspector/README.md b/tools/ui_inspector/README.md index 2955324..207b8c1 100644 --- a/tools/ui_inspector/README.md +++ b/tools/ui_inspector/README.md @@ -1,14 +1,34 @@ -# OpenClaw UI Inspector（桌面试验版 v0.8.1） +# OpenClaw UI Inspector（UI Automation Recorder v0.16.0） -> **有焦点的窗口 = 采集目标**（与大小无关）。 -> 企微暖树：**只调用** `jiangchang_desktop_wake.ensure_ui_ready`（与技能同路径），不复制注入逻辑。 -> 主窗 → profile `wecom`；搜索浮层 `SearchResultWindow2` → profile `wecom_search`（需 kit≥1.2.6）。 +默认窗口采用窄长控制台布局：停靠到鼠标所在显示器右侧，顶部预留目标应用标题栏安全区；也可在界面中一键切换到左侧。 + +> **定位**：以人工操作为主线的开发期录制器，产出可交给 AI 编程工具的 Action Flow + Element Catalog。 +> **不是**：每 2 秒无条件 dump 页面的流水线（该行为仅保留为 Legacy 模式）。 + +## 核心模型 + +```text +人工操作 + → Raw Event Journal（先落事实，不阻塞） + → Action Reducer（去重、输入事务、pointerdown/click 合并） + → Target Resolver（距离 + 面积 + 语义） + → ActionRecord + Element Catalog + → 异步合并式 Observation / State / Screenshot +``` + +| 概念 | 作用 | +|------|------| +| **Action** | 流程主线（click / fill / press_key / checkpoint …） | +| **State** | 动作前后上下文与证据（tree/index/screenshot） | +| **Element Catalog** | 开发期与运行期共用的元素资产 | +| **Locator** | 结构化属性 + 编译后的 expression；须验证匹配数 | +| **原始 dump** | 诊断证据；不再把每次 heartbeat 登记为流程步骤 | +| **Raw Event Journal** | 可重算的原始事实；Reducer 出错时无需重走人工流程 | ## 启动 ```powershell cd D:\OpenClaw\client-commons\skill-template -# 建议使用已安装 jiangchang-platform-kit>=1.2.5 的 Python（与宿主共享 venv 一致） python tools/ui_inspector/run.py --scene demo ``` @@ -16,34 +36,154 @@ python tools/ui_inspector/run.py --scene demo ```powershell pip install -r tools/ui_inspector/requirements.txt -# 企微自动暖树（强烈建议） -pip install "jiangchang-platform-kit>=1.2.5" # 或指向你们 Gitea/本地 index +pip install "jiangchang-platform-kit>=1.2.11" ``` -## 规则 +## 录制流程 -1. 焦点在谁就采谁（排除 Inspector 自身 hwnd） -2. **自动拾取默认勾选**；点「开始流程」后先 **5 秒倒数**，请在此期间点击目标窗 -3. Inspector **保持置顶**便于看进度；置顶 ≠ 焦点，不因此改采本面板 -4. **不强制最大化**目标窗，保持操作者当时的大小/位置（仅最小化时会还原） -5. 不会因为窗口小 / FindAll 少而改采别的软件 -6. **企微自动暖树（无感）**：识别到 `WeWorkWindow` 时，在准备/拾取前静默调用 - `ensure_ui_ready("wecom")`；已暖则快速返回。结果写入每步 `meta.json` → `dump_stats.kit_wake` +主界面固定使用自动动作录制，只保留 **开始 / 暂停（继续）/ 结束**。Recorder 立即记录 Action，完整证据由独立后台队列异步合并采集。Inspector 自身窗口上的点击和输入不会写入目标 Action Flow。 -## 空壳说明 +“运行状态与拾取结果”会对浏览器和桌面统一采用单步骤覆盖式显示；新动作替换上一动作，同一动作的异步证据返回后在原位置更新定位器、唯一性或失败原因。迟到的旧动作证据不会覆盖更新的步骤，完整历史始终保存在 `action_flow.json`、`actions/` 与 `element_catalog.json` 中。 -- **已装 kit**：冷企微应被 n2-wake 暖到 `findall` 较大、`provider_cold=false`。若仍冷，看状态栏 `kit_wake: FAILED …`。 -- **未装 kit**：行为与旧版相同，可能 `findall≈2`、`provider_cold=true`；状态栏会提示未安装 platform-kit。 +引导、手动检查点和 Legacy 间隔 dump 仅作为底层兼容与开发诊断能力保留，不进入普通用户流程；旧偏好会自动迁移回自动录制。 -## 验收建议（企微） +## 运行时稳定性 -1. 打开并登录桌面企业微信 -2. 启动 Inspector →「开始流程」→ 倒数内点到企微主窗 -3. 看详情区 / 第一步 `meta.json`：`kit_wake.warm=true`，`findall` 通常 ≫ 80 -4. 再搜联系人、开会话，确认 `groupbuddyname` 的 Value 能采到 +- Tk 主线程只处理界面；导航、Dump、截图、动作落盘均在后台执行。 +- 后台结果通过 UI 消息队列回传，后台线程不读取 Tk Variable 或 Widget。 +- 原始事件先追加到 `journal/raw_events.jsonl`，再进入语义 Action FIFO。 +- Action 热路径不执行完整页面 dump；慢速证据在独立队列按页面合并采集。 +- 异步证据标记为 `coalesced_eventual`，不会伪装成事件时刻的精确 after state。 +- Playwright 命令均有超时；导航失败不会被伪装成“已就绪”。 +- 浏览器使用 context 级事件桥并保留轮询回退，整页跳转前的 click 不随旧 document 丢失。 -## 冒烟（不弹 GUI） +## 拾取通道 + +- **浏览器**：Inspector 自管 Playwright；起始 URL 可选；Browser Recorder（捕获阶段监听，不 preventDefault） + - 动作后的 Dump / 截图绑定事件来源 `page_id`；手动检查点才按当前可见标签页回退，不再固定 `pages[0]` + - 列表页新开详情标签后，切到详情再 dump/录制即可；关闭详情回到列表同理 + - Recorder 会在所有已打开标签上安装监听并汇总事件 + - 输入框、`contenteditable`、SVG/图标点击会沿 `composedPath` 提升到真实可操作目标；原始命中节点仍保留作诊断证据 + - 语义定位不足时生成完整 CSS 路径，并在事件来源页面实时验证唯一性后才进入 Locator Catalog + - 链接保留 `href`，可编辑区生成稳定祖先作用域选择器；事件时刻的唯一性计数随快照保存，避免跳转后误判 + - `press_key` 保留原始事件、Page/Document 身份和目标元素引用，不再依赖前一步推断焦点 + - 页面跳转采用短暂因果重排，确保触发跳转的 click 先于 navigation；新标签同时保留 opener `page_id` + - 同时监听 `pointerdown` 与 `click`：正常 click 会去重；页面提前卸载时 pointerdown 作为保底 Action + - 通用目标评分限制祖先距离与面积，不会把小图标提升成整页容器 + - 目标解析联合考虑语义、几何、事件时刻唯一性和定位器稳定性；SVG/path 仅作命中证据，优先选择唯一的可点击包装节点 + - 输入事务与 Element Catalog 共用稳定元素身份，焦点/校验/悬停 class 变化不会拆分一次连续输入 + - 输入事务使用浏览器源时间并保存全部 `raw_event_ids`，支持从原始日志审计和重算 + - 完整 Dump / Screenshot 由后台 Evidence Coordinator 合并执行；手动检查点仍保留稳定采集 +- **桌面**：UIA；Windows Recorder 使用进程外 `WH_MOUSE_LL` / `WH_KEYBOARD_LL` 事件监听，点击后由独立线程执行 `ControlFromPoint`。Hook 不注入目标进程；`GetAsyncKeyState` 仅作为 Hook 无法安装时的降级路径。 + - 输入使用焦点控件 Value/Text 变化聚合为一次 `fill`，不保存中文输入法的拼音按键；混合 DPI 应用会在键盘阶段重新探测正确坐标空间。 + - 已注册的桌面适配器会在宣布“录制就绪”前完成必要的 Accessibility warm；企业微信首次启动可能多等十几秒，但不会再让首个输入事务与后台暖树竞争。 + - 输入法候选/组合窗口只作为临时系统 UI 忽略，不会替换业务输入框；编辑器内的 Backspace/Delete 合并进最终 `fill`。 + - `Enter`、`Tab` 等功能键优先继承刚才的文本编辑器目标，不再退化为整个根窗口。 + - 鼠标按下时会枚举光标下可见顶层窗口，处理 owner window 吞掉浮窗命中的应用；后置树再把搜索/列表大容器提升到坐标命中的具体行。 + - 超过 5 秒的延迟证据不能用动态消息摘要/预览改写旧点击；坐标不可靠且只有一个可见结果行时，才使用唯一结果行作为保守回退。 + - 原始键盘活动只写入不含内容的事务标记；若发生输入但没有生成 `fill/type`，质量检查会明确告警。 +- **手机**：未实现，不得宣称已支持 + +## 输出目录（双写兼容） + +```text +_captures/_/ + flow.json # v1 schema；可选 v2 指针字段 + SUMMARY.md + action_flow.json # v2 + element_catalog.json # v2 + pages.json # Page / Document 记录 + journal/raw_events.jsonl + diagnostics/metrics.json + diagnostics/validation.json + diagnostics/replay_report.json # 静态、非破坏性的回放就绪预检 + steps/NN_xxx/ # v1：meta/tree/index/locatable/screen/SUMMARY + states// # v2：证据 + observations// # v2：每次真实采集证据（不去重） + actions/.json + elements/.json +``` + +旧读取方只认 `flow.json` + `steps/` 仍可工作。`flow.json` 中的 `v2` 字段可忽略。 + +### Schema 名 + +- `openclaw.ui_inspector.state.v1` +- `openclaw.ui_inspector.observation.v1` +- `openclaw.ui_inspector.action.v1` +- `openclaw.ui_inspector.element.v1` +- `openclaw.ui_inspector.element_catalog.v1` +- `openclaw.ui_inspector.action_flow.v1` +- `openclaw.ui_inspector.flow.v1`（原有） + +## 交给 AI 编程工具 + +优先提供： + +1. `action_flow.json` — 动作序列、目标和证据关联；异步证据不冒充精确 after state +2. `element_catalog.json` — logical_name、结构化 locator 候选、唯一性结果 +3. 需要排障时再附对应 `states//` 或旧 `steps/` 下的 tree/locatable/screen + +不要只丢一堆无动作语义的定时 dump。 + +`diagnostics.capture_validator` 会报告缺失的生产级定位器、目标元素引用、原始事件关联、根窗口按键目标、列表大容器点击及丢失的输入事务，避免“动作录到了”被误判为“可以稳定生成代码”。 + +## 当前能力边界 + +| 能力 | 状态 | +|------|------| +| click | 已实现（browser + windows） | +| fill/type 聚合 | 已实现 | +| press_key（Enter/Tab/Esc 等） | 已实现 | +| checkpoint | 已实现 | +| navigation | 部分（URL 变化观察） | +| 其余动作类型 | Schema 预留，`capability=schema_only` | +| 手机录制 | **未实现** | +| 截图像素级脱敏 | 未实现（敏感页在 meta 标记 `may_contain_sensitive_data`） | + +## 敏感信息 + +- 密码框 / token / cookie / 手机号 / 邮箱等会脱敏，不写入 action_flow / catalog 明文。 +- 截图无法可靠脱敏时标记风险；可用配置关闭敏感页截图保存。 +- **禁止**在测试或产物中使用真实客户数据。 + +## WeCom Adapter + +企业微信专用规则在 `adapters/wecom.py`： + +- `WeWorkWindow` → profile `wecom` +- `SearchResultWindow2` → profile `wecom_search` +- Warm **cache key = profile + hwnd**，主窗 warm 不会假装搜索浮层已暖 +- Recorder **捕获点击目标前不执行全量 warm**；warm 属于 dump/checkpoint/live 验证路径 + +## Legacy 间隔模式 + +Legacy 能力仅供底层兼容与开发排障，不在标准界面暴露。 +默认动作驱动模式下，heartbeat 只做输入聚合刷新，**相同 fingerprint 不新增 State，不产生 Action**。 + +## 冒烟与单测 ```powershell python tools/ui_inspector/smoke_check.py +python -m pytest tools/ui_inspector/tests -q ``` + +## 目录结构 + +```text +ui_inspector/ + core/ # 三端共用模型、管线、策略 + surfaces/ # browser / windows / future mobile + evidence/ # 异步证据协调 + locators/ # Locator Catalog 与稳定性 + storage/ # 新旧采集格式读写 + diagnostics/ # 指标与通用质量检查 + compatibility/ # v1/legacy 兼容入口 + ui/ # 新 UI facade；app.py 暂保留为 Tk 兼容入口 + adapters/ # 可选产品增强，不参与核心正确性 + tests/ # unit / contract / integration / e2e +``` + +`models/`、`recorder/`、`drivers/` 等旧路径暂时保留为兼容实现；新入口已从 `surfaces/`、`core/` 和 `storage/` 访问，后续可按模块逐步内收而不破坏现有技能。 + +暖树：`ensure_ui_ready`（kit≥1.2.6）；整页 dump：kit≥1.2.11。 diff --git a/tools/ui_inspector/__init__.py b/tools/ui_inspector/__init__.py index 3aa7dfd..08d010e 100644 --- a/tools/ui_inspector/__init__.py +++ b/tools/ui_inspector/__init__.py @@ -1,3 +1,3 @@ -"""Desktop UI Inspector — development-only UIA capture tool (additive, disposable).""" +"""UI Inspector — development-only multi-surface capture (windows UIA + browser Playwright).""" -__version__ = "0.8.1" +__version__ = "0.16.0" diff --git a/tools/ui_inspector/adapters/__init__.py b/tools/ui_inspector/adapters/__init__.py new file mode 100644 index 0000000..9c7deda --- /dev/null +++ b/tools/ui_inspector/adapters/__init__.py @@ -0,0 +1,7 @@ +"""App adapters registry (WeCom-specific rules live here, not in generic layers).""" + +from .base import AppAdapter +from .registry import get_adapter, register_adapter +from . import wecom as _wecom # noqa: F401 — register on import + +__all__ = ["AppAdapter", "get_adapter", "register_adapter"] diff --git a/tools/ui_inspector/adapters/base.py b/tools/ui_inspector/adapters/base.py new file mode 100644 index 0000000..76d5793 --- /dev/null +++ b/tools/ui_inspector/adapters/base.py @@ -0,0 +1,17 @@ +"""Adapter base protocol.""" + +from __future__ import annotations + +from typing import Any, Optional, Protocol + + +class AppAdapter(Protocol): + name: str + + def matches(self, *, class_name: str = "", title: str = "", process_name: str = "") -> bool: ... + + def wake_profile(self, *, class_name: str = "", title: str = "") -> Optional[str]: ... + + def cache_key(self, *, profile: str, hwnd: int = 0) -> str: ... + + def propose_postconditions(self, changes: list[dict[str, Any]]) -> list[dict[str, Any]]: ... diff --git a/tools/ui_inspector/adapters/registry.py b/tools/ui_inspector/adapters/registry.py new file mode 100644 index 0000000..5ce07f5 --- /dev/null +++ b/tools/ui_inspector/adapters/registry.py @@ -0,0 +1,31 @@ +"""Adapter registry.""" + +from __future__ import annotations + +from typing import Any, Optional + +_REGISTRY: list[Any] = [] + + +def register_adapter(adapter: Any) -> Any: + _REGISTRY.append(adapter) + return adapter + + +def get_adapter( + *, + class_name: str = "", + title: str = "", + process_name: str = "", +) -> Optional[Any]: + for ad in _REGISTRY: + try: + if ad.matches(class_name=class_name, title=title, process_name=process_name): + return ad + except Exception: + continue + return None + + +def all_adapters() -> list[Any]: + return list(_REGISTRY) diff --git a/tools/ui_inspector/adapters/wecom.py b/tools/ui_inspector/adapters/wecom.py new file mode 100644 index 0000000..3fa2f53 --- /dev/null +++ b/tools/ui_inspector/adapters/wecom.py @@ -0,0 +1,103 @@ +"""WeCom (企业微信) adapter — profile / window / postcondition helpers.""" + +from __future__ import annotations + +from typing import Any, Optional + +from .registry import register_adapter + +PROFILE_WECOM = "wecom" +PROFILE_WECOM_SEARCH = "wecom_search" + +_THRESHOLDS = { + PROFILE_WECOM: 80, + PROFILE_WECOM_SEARCH: 20, +} + + +class WeComAdapter: + name = "wecom" + + def matches( + self, *, class_name: str = "", title: str = "", process_name: str = "" + ) -> bool: + return self.wake_profile(class_name=class_name, title=title) is not None or ( + "WXWork" in (process_name or "") + ) + + def wake_profile(self, *, class_name: str = "", title: str = "") -> Optional[str]: + cls = (class_name or "").strip() + title = title or "" + if cls == "SearchResultWindow2": + return PROFILE_WECOM_SEARCH + if cls == "WeWorkWindow": + return PROFILE_WECOM + if "WeWork" in cls and ( + "企业微信" in title or "WeCom" in title or "wecom" in title.lower() + ): + return PROFILE_WECOM + return None + + def cache_key(self, *, profile: str, hwnd: int = 0) -> str: + """Separate main window vs SearchResultWindow2 warm caches.""" + return f"{profile}:hwnd:{int(hwnd or 0)}" + + def warm_threshold(self, profile: str) -> int: + return int(_THRESHOLDS.get(profile, 80)) + + def is_search_popup(self, *, class_name: str = "") -> bool: + return (class_name or "") == "SearchResultWindow2" + + def is_main_window(self, *, class_name: str = "") -> bool: + return (class_name or "") == "WeWorkWindow" + + def known_ids(self) -> dict[str, str]: + return { + "searchedit": "search_box", + "groupbuddyname": "contact_row_name", + "search_name": "search_result_name", + "send_rich_edit": "compose_box", + "panel_title_label": "panel_title", + } + + def propose_postconditions(self, changes: list[dict[str, Any]]) -> list[dict[str, Any]]: + props: list[dict[str, Any]] = [] + for ch in changes or []: + kind = ch.get("kind") + detail = ch.get("detail") or {} + if kind == "popup_disappeared" and "SearchResult" in str( + detail.get("class_name") or "" + ): + props.append( + { + "description": "SearchResultWindow2 disappeared", + "kind": "window_absent", + "status": "proposed", + "detail": detail, + } + ) + if kind == "element_appeared": + name = str(detail.get("name") or detail.get("automation_id") or "") + if name in {"send_rich_edit", "main_chattext", "panel_title_label"}: + props.append( + { + "description": f"WeCom element {name} appeared", + "kind": "element_present", + "status": "proposed", + "detail": detail, + } + ) + if kind == "title_changed": + props.append( + { + "description": f"Chat title became {detail.get('to')!r}", + "kind": "title_equals", + "status": "proposed", + "detail": detail, + } + ) + return props + + +wecom_adapter = WeComAdapter() +register_adapter(wecom_adapter) diff --git a/tools/ui_inspector/app.py b/tools/ui_inspector/app.py index 440bde6..67f4611 100644 --- a/tools/ui_inspector/app.py +++ b/tools/ui_inspector/app.py @@ -1,15 +1,23 @@ -"""Tk panel v0.8.1: 开始流程 → 手动拾取 / 自动串行循环 → 结束流程。 +"""Tk panel v0.14: Action-driven UI Automation Recorder (compatible with v1 dumps). -企微主窗 / 搜索浮层在识别/拾取前静默调用 kit ensure（wecom / wecom_search）。 +拾取模式（单选，记忆上次）：浏览器 / 桌面 / 手机。 +- 浏览器：强制 Playwright；起始 URL 自动打开；Browser Recorder +- 桌面：强制 UIA；Windows Recorder（进程外轮询，无注入） +- 手机：占位未实现 + +录制模式：自动录制（默认）/ 引导录制 / 手动检查点；legacy 间隔 dump 可选。 """ from __future__ import annotations import argparse +import ctypes import os +import queue import subprocess import sys import threading +from ctypes import wintypes from pathlib import Path import tkinter as tk from tkinter import messagebox, simpledialog, ttk @@ -20,6 +28,13 @@ if str(_HERE) not in sys.path: import artifacts as art # noqa: E402 import desktop_uia as uia # noqa: E402 +import kit_browser # noqa: E402 +import skip_policy # noqa: E402 +import target # noqa: E402 +import user_prefs # noqa: E402 +from surfaces.registry import get_driver # noqa: E402 +from flow_session import FlowSession # noqa: E402 +from drivers.base import DumpResult # noqa: E402 try: from . import __version__ @@ -33,17 +48,130 @@ except Exception: _mod = importlib.util.module_from_spec(_spec) # type: ignore[arg-type] assert _spec and _spec.loader _spec.loader.exec_module(_mod) - __version__ = getattr(_mod, "__version__", "0.8.1") + __version__ = getattr(_mod, "__version__", "0.16.0") except Exception: - __version__ = "0.8.1" + __version__ = "0.16.0" -# Auto loop: after each dump finishes, wait this long, then dump again. +# Legacy interval mode: after each dump finishes, wait this long, then dump again. _AUTO_GAP_MS = 2000 +# Heartbeat for action-driven mode (flush idle input; do NOT create steps). +_HEARTBEAT_MS = 800 # After frontmost target is resolved, short delay before first dump. _AUTO_START_DELAY_MS = 400 # After「开始流程」with auto-pick: countdown so user can focus the target app. _START_COUNTDOWN_SEC = 5 _FRONTMOST_WAIT_SEC = 15.0 +_STABILITY_TIMEOUT_SEC = 3.0 + +# Recording modes +_MODE_AUTO = "auto" +_MODE_GUIDED = "guided" +_MODE_CHECKPOINT = "checkpoint" +_MODE_LEGACY = "legacy_interval" + +_WINDOW_RIGHT_MARGIN = 16 +_WINDOW_BOTTOM_MARGIN = 16 +_WINDOW_MIN_WIDTH = 520 +_WINDOW_MIN_HEIGHT = 600 + +# Default flow-name slugs aligned with the pick-mode radio labels. +# Switching mode only auto-updates the field when it still holds one of these. +_MODE_FLOW_NAMES = { + target.SURFACE_BROWSER: "browser", + target.SURFACE_WINDOWS: "desktop", + target.SURFACE_ANDROID: "mobile", +} +_DEFAULT_FLOW_NAME_SLUGS = frozenset(_MODE_FLOW_NAMES.values()) + + +def _flow_name_for_mode(mode: str) -> str: + return _MODE_FLOW_NAMES.get(mode, "desktop") + + +def _clamp(value: int, low: int, high: int) -> int: + return max(low, min(high, value)) + + +def _default_window_geometry( + work_area: tuple[int, int, int, int], + *, + side: str = "right", +) -> tuple[int, int, int, int]: + """Return a narrow console geometry inside a monitor work area. + + The top gap deliberately leaves the target application's title-bar controls + unobstructed. Coordinates may be negative for monitors left of the primary. + """ + + left, top, right, bottom = work_area + work_width = max(1, right - left) + work_height = max(1, bottom - top) + width = _clamp(round(work_width * 0.30), 560, 620) + width = min(width, max(420, work_width - 32)) + top_gap = _clamp(round(work_height * 0.07), 64, 88) + height = max(480, work_height - top_gap - _WINDOW_BOTTOM_MARGIN) + x = ( + left + _WINDOW_RIGHT_MARGIN + if side == "left" + else right - width - _WINDOW_RIGHT_MARGIN + ) + y = top + top_gap + return width, height, x, y + + +def _format_geometry(width: int, height: int, x: int, y: int) -> str: + x_part = f"+{x}" if x >= 0 else str(x) + y_part = f"+{y}" if y >= 0 else str(y) + return f"{width}x{height}{x_part}{y_part}" + + +def _format_live_action(item: dict) -> str: + """Compact, surface-neutral live action text for the result panel.""" + labels = { + "click": "点击", "fill": "输入", "type": "输入", + "press_key": "按键", "navigation": "导航", "checkpoint": "检查点", + } + phases = { + "captured": "已捕获，证据采集中", + "evidence": "证据已更新", + "evidence_failed": "证据采集失败", + } + action_type = str(item.get("action_type") or "unknown") + lines = [ + f"#{item.get('sequence')} {labels.get(action_type, action_type)} · " + f"{phases.get(str(item.get('phase') or ''), str(item.get('phase') or ''))}", + f" 表面: {item.get('surface') or '-'} 目标: " + f"{item.get('target_name') or '-'} ({item.get('target_type') or '-'})", + ] + if item.get("input_summary"): + lines.append(f" 输入: {item['input_summary']}") + if item.get("window_class"): + lines.append( + f" 窗口: {item.get('window_title') or '-'} [{item.get('window_class')}]" + ) + if item.get("locator"): + lines.append( + f" 定位: {item['locator']} match={item.get('match_count')} " + f"unique={item.get('unique')}" + ) + elif item.get("phase") == "evidence_failed": + lines.append(f" 错误: {item.get('error') or '未知错误'}") + else: + lines.append(" 定位: 等待异步证据") + return "\n".join(lines) + + +def _merge_current_live_action(current: dict | None, incoming: dict) -> dict | None: + """Keep the newest action while allowing its evidence phase to replace it.""" + if not str(incoming.get("id") or ""): + return current + if current is None: + return dict(incoming) + current_sequence = int(current.get("sequence") or 0) + incoming_sequence = int(incoming.get("sequence") or 0) + if incoming_sequence < current_sequence: + return current + return dict(incoming) class InspectorApp: @@ -51,7 +179,7 @@ class InspectorApp: self, *, captures_root: Path, - scene: str = "desktop", + scene: str = "", max_depth: int = 16, max_nodes: int = 8000, ) -> None: @@ -64,102 +192,386 @@ class InspectorApp: self.flow: dict | None = None self.last_window: dict | None = None self.sticky_pid: int | None = None - self._warmed_pids: set[int] = set() + # Browser entries use pid; native entries use (pid, window class) so a + # WeCom SearchResultWindow2 is not mistaken for its already-warmed main + # WeWorkWindow merely because both belong to the same process. + self._warmed_pids: set[object] = set() self._busy = False self._auto_loop_active = False self._pending_end = False self._auto_after_id: str | None = None self._countdown_after_id: str | None = None self._countdown_left = 0 + self._session: FlowSession | None = None + self._desktop_entry_dump: DumpResult | None = None + self._heartbeat_after_id: str | None = None + self._rec_paused = False + self._current_live_action: dict | None = None + self._ui_queue: queue.Queue[tuple[object, tuple, dict]] = queue.Queue() + self._ui_poll_after_id: str | None = None + self._closed = False + + prefs = user_prefs.load_prefs() + initial_mode = str(prefs.get("mode") or user_prefs.DEFAULT_MODE) + if initial_mode in {"mobile", "phone", "ios"}: + initial_mode = target.SURFACE_ANDROID + if initial_mode not in { + target.SURFACE_BROWSER, + target.SURFACE_WINDOWS, + target.SURFACE_ANDROID, + }: + initial_mode = user_prefs.DEFAULT_MODE + scene_name = (scene or "").strip() + initial_flow_name = scene_name or _flow_name_for_mode(initial_mode) self.root = tk.Tk() - self.root.title(f"OpenClaw UI Inspector v{__version__} (desktop)") - self.root.geometry("840x660") - self.root.minsize(720, 540) + self.root.title(f"OpenClaw UI Inspector v{__version__}") + self._dock_side = "right" + self._apply_default_geometry() self.root.attributes("-topmost", True) - self.flow_name_var = tk.StringVar(value=self.scene) + self.flow_name_var = tk.StringVar(value=initial_flow_name) + self.mode_var = tk.StringVar(value=initial_mode) + self.start_url_var = tk.StringVar(value=str(prefs.get("start_url") or "")) + # Standard UI always records actions automatically. Legacy strategies + # remain supported internally for old captures and developer tooling. self.auto_pick_var = tk.BooleanVar(value=True) + self.recording_mode_var = tk.StringVar(value=_MODE_AUTO) + # 默认开：悬停即可采搜索浮层，不必点进会话；均可随时取消勾选回退旧行为 + self.under_cursor_var = tk.BooleanVar(value=True) + self.prefer_wecom_search_var = tk.BooleanVar(value=True) self.flow_var = tk.StringVar(value="流程：未开始（请先点「开始流程」）") - self.target_var = tk.StringVar(value="尚未拾取") + self.target_var = tk.StringVar(value="等待开始流程") + self.rec_stats_var = tk.StringVar( + value="recorder: idle | actions=0 states=0 observations=0 elements=0" + ) self.status_var = tk.StringVar( - value="用法：开始流程 →（默认自动拾取）5 秒倒数 → 串行 dump。" - "企微将自动 ensure_ui_ready（需 platform-kit≥1.2.5）；选窗看焦点，不强制放大。" + value="选择模式后点击「开始流程」。" ) self._build_ui() + for var in ( + self.flow_var, + self.rec_stats_var, + self.target_var, + self.status_var, + ): + var.trace_add("write", self._on_status_var_changed) + self._render_status_summary() + self._refresh_mode_widgets() self.root.protocol("WM_DELETE_WINDOW", self._on_close) + self._schedule_ui_queue_poll() + + def _current_monitor_work_area(self) -> tuple[int, int, int, int]: + """Get the work area of the monitor containing the mouse pointer.""" + + if sys.platform == "win32": + try: + class MONITORINFO(ctypes.Structure): + _fields_ = [ + ("cbSize", wintypes.DWORD), + ("rcMonitor", wintypes.RECT), + ("rcWork", wintypes.RECT), + ("dwFlags", wintypes.DWORD), + ] + + point = wintypes.POINT() + user32 = ctypes.windll.user32 + user32.GetCursorPos.argtypes = [ctypes.POINTER(wintypes.POINT)] + user32.GetCursorPos.restype = wintypes.BOOL + user32.MonitorFromPoint.argtypes = [wintypes.POINT, wintypes.DWORD] + user32.MonitorFromPoint.restype = wintypes.HANDLE + user32.GetMonitorInfoW.argtypes = [ + wintypes.HANDLE, + ctypes.POINTER(MONITORINFO), + ] + user32.GetMonitorInfoW.restype = wintypes.BOOL + if user32.GetCursorPos(ctypes.byref(point)): + monitor = user32.MonitorFromPoint(point, 2) + info = MONITORINFO() + info.cbSize = ctypes.sizeof(info) + if monitor and user32.GetMonitorInfoW(monitor, ctypes.byref(info)): + area = info.rcWork + return area.left, area.top, area.right, area.bottom + except Exception: + pass + return 0, 0, self.root.winfo_screenwidth(), self.root.winfo_screenheight() + + def _apply_default_geometry(self) -> None: + width, height, x, y = _default_window_geometry( + self._current_monitor_work_area(), side=self._dock_side + ) + self.root.geometry(_format_geometry(width, height, x, y)) + self.root.minsize(min(_WINDOW_MIN_WIDTH, width), min(_WINDOW_MIN_HEIGHT, height)) + + def _toggle_dock(self) -> None: + self._dock_side = "left" if self._dock_side == "right" else "right" + self._apply_default_geometry() + self.dock_btn.configure(text="靠右" if self._dock_side == "left" else "靠左") + + def _on_status_var_changed(self, *_args: object) -> None: + self._render_status_summary() + + def _render_status_summary(self) -> None: + if not hasattr(self, "status_text"): + return + if not self.flow: + headline = "● 未开始 · recorder idle" + else: + v2 = self.flow.get("v2") or {} + live = self._session.stats() if self._session else {} + status = str(self.flow.get("status") or "unknown") + status_name = {"open": "录制中", "ended": "已结束"}.get(status, status) + action_count = live.get("action_count", v2.get("action_count", 0)) + state_count = live.get("state_count", v2.get("state_count", 0)) + dropped = live.get("dropped_events", 0) + input_health = live.get("input_backend") or {} + input_suffix = "" + if input_health: + input_suffix = ( + f" · input {input_health.get('raw_input_events', 0)}" + f" · {input_health.get('backend', 'unknown')}" + ) + headline = ( + f"● {status_name} · actions {action_count} · states {state_count}" + f" · dropped {dropped}{input_suffix}" + ) + target_line = self.target_var.get().strip() + status_line = self.status_var.get().strip() + lines = [headline] + if target_line: + lines.append(target_line) + if status_line and status_line != target_line: + lines.append(status_line) + self.status_text.configure(state=tk.NORMAL) + self.status_text.delete("1.0", tk.END) + self.status_text.insert(tk.END, "\n".join(line for line in lines if line)) + self.status_text.configure(state=tk.DISABLED) + + def _post_ui(self, callback, *args, **kwargs) -> None: + """Worker-safe UI handoff; workers must never call Tk directly.""" + self._ui_queue.put((callback, args, kwargs)) + + def _schedule_ui_queue_poll(self) -> None: + self._ui_poll_after_id = self.root.after(40, self._drain_ui_queue) + + def _drain_ui_queue(self) -> None: + self._ui_poll_after_id = None + for _ in range(100): + try: + callback, args, kwargs = self._ui_queue.get_nowait() + except queue.Empty: + break + try: + callback(*args, **kwargs) # type: ignore[operator] + except Exception as exc: + try: + self._set_status(f"后台结果处理失败: {exc}") + except Exception: + pass + try: + if self.root.winfo_exists(): + self._schedule_ui_queue_poll() + except Exception: + pass + + def _current_mode(self) -> str: + mode = (self.mode_var.get() or user_prefs.DEFAULT_MODE).strip().lower() + if mode in {"mobile", "phone", "ios"}: + return target.SURFACE_ANDROID + if mode not in { + target.SURFACE_BROWSER, + target.SURFACE_WINDOWS, + target.SURFACE_ANDROID, + }: + return user_prefs.DEFAULT_MODE + return mode + + def _persist_prefs(self) -> None: + user_prefs.save_prefs( + mode=self._current_mode(), + start_url=(self.start_url_var.get() or "").strip(), + recording_mode=_MODE_AUTO, + ) + + def _current_recording_mode(self) -> str: + return _MODE_AUTO + + def _is_legacy_interval(self) -> bool: + return self._current_recording_mode() == _MODE_LEGACY + + def _on_mode_change(self) -> None: + self._persist_prefs() + self._sync_default_flow_name() + self._refresh_mode_widgets() + mode = self._current_mode() + if mode == target.SURFACE_BROWSER: + self._set_status( + "已选「浏览器自动化」：开始流程后将打开起始 URL 并 dump Playwright 页。" + ) + elif mode == target.SURFACE_WINDOWS: + self._set_status( + "已选「桌面自动化」：开始流程后按光标/前台窗口做 UIA dump（不启浏览器）。" + ) + else: + self._set_status("已选「手机自动化」：尚未实现，开始流程时会提示。") + + def _sync_default_flow_name(self) -> None: + """Keep flow name aligned with pick mode unless the user customized it.""" + current = (self.flow_name_var.get() or "").strip() + if current and current not in _DEFAULT_FLOW_NAME_SLUGS: + return + self.flow_name_var.set(_flow_name_for_mode(self._current_mode())) + + def _refresh_mode_widgets(self) -> None: + mode = self._current_mode() + is_browser = mode == target.SURFACE_BROWSER + is_desktop = mode == target.SURFACE_WINDOWS + try: + if is_browser: + self.url_row.grid() + else: + self.url_row.grid_remove() + except Exception: + pass + try: + if is_desktop: + self.desktop_options.pack( + fill=tk.X, + padx=8, + pady=(0, 4), + before=self.result_panel, + ) + else: + self.desktop_options.pack_forget() + except Exception: + pass + + def _synthetic_browser_info(self) -> uia.WindowInfo: + return uia.WindowInfo( + hwnd=0, + pid=0, + title="Playwright", + class_name="PlaywrightPage", + process_name="chrome.exe", + ) def _build_ui(self) -> None: - pad = {"padx": 8, "pady": 4} + pad = {"padx": 8, "pady": (4, 2)} top = ttk.Frame(self.root) - top.pack(fill=tk.X, **pad) - ttk.Label(top, text="流程名").pack(side=tk.LEFT) - ttk.Entry(top, textvariable=self.flow_name_var, width=28).pack(side=tk.LEFT, padx=4) - ttk.Button(top, text="打开输出目录", command=self.on_open_out).pack(side=tk.LEFT, padx=8) + top.pack(fill=tk.X, padx=8, pady=(8, 2)) + top.columnconfigure(1, weight=1) + ttk.Label(top, text="流程名").grid(row=0, column=0, sticky=tk.W) + ttk.Entry(top, textvariable=self.flow_name_var).grid( + row=0, column=1, sticky=tk.EW, padx=(4, 6) + ) + ttk.Button(top, text="打开目录", command=self.on_open_out).grid( + row=0, column=2, padx=(0, 4) + ) + self.dock_btn = ttk.Button(top, text="靠左", width=6, command=self._toggle_dock) + self.dock_btn.grid(row=0, column=3) - btns = ttk.Frame(self.root) + mode_box = ttk.LabelFrame(self.root, text="拾取模式") + mode_box.pack(fill=tk.X, **pad) + mode_box.columnconfigure(0, weight=1) + mode_row = ttk.Frame(mode_box) + mode_row.grid(row=0, column=0, sticky=tk.EW, padx=6, pady=(2, 0)) + for column in range(3): + mode_row.columnconfigure(column, weight=1) + ttk.Radiobutton( + mode_row, + text="浏览器自动化", + value=target.SURFACE_BROWSER, + variable=self.mode_var, + command=self._on_mode_change, + ).grid(row=0, column=0, sticky=tk.W, padx=4) + ttk.Radiobutton( + mode_row, + text="桌面自动化", + value=target.SURFACE_WINDOWS, + variable=self.mode_var, + command=self._on_mode_change, + ).grid(row=0, column=1, sticky=tk.W, padx=4) + ttk.Radiobutton( + mode_row, + text="手机自动化", + value=target.SURFACE_ANDROID, + variable=self.mode_var, + command=self._on_mode_change, + ).grid(row=0, column=2, sticky=tk.W, padx=4) + + self.url_row = ttk.Frame(mode_box) + self.url_row.grid(row=1, column=0, sticky=tk.EW, padx=6, pady=(2, 4)) + self.url_row.columnconfigure(1, weight=1) + ttk.Label(self.url_row, text="起始 URL").grid(row=0, column=0, sticky=tk.W) + ttk.Entry(self.url_row, textvariable=self.start_url_var).grid( + row=0, column=1, sticky=tk.EW, padx=(6, 0) + ) + + btns = ttk.LabelFrame(self.root, text="流程控制") btns.pack(fill=tk.X, **pad) - self.start_btn = ttk.Button(btns, text="1. 开始流程", command=self.on_start_flow) - self.start_btn.pack(side=tk.LEFT, padx=2) - self.pick_btn = ttk.Button(btns, text="2. 拾取当前页", command=self.on_pick_page) - self.pick_btn.pack(side=tk.LEFT, padx=2) - self.end_btn = ttk.Button(btns, text="3. 结束流程", command=self.on_end_flow) - self.end_btn.pack(side=tk.LEFT, padx=2) - self.auto_chk = ttk.Checkbutton( - btns, - text="自动拾取（串行·步间2秒）", - variable=self.auto_pick_var, - command=self.on_auto_toggle, - ) - self.auto_chk.pack(side=tk.LEFT, padx=12) + for column in range(3): + btns.columnconfigure(column, weight=1, uniform="flow_buttons") + self.start_btn = ttk.Button(btns, text="开始", command=self.on_start_flow) + self.start_btn.grid(row=0, column=0, sticky=tk.EW, padx=(6, 2), pady=(3, 6)) + self.pause_btn = ttk.Button(btns, text="暂停", command=self.on_pause_resume) + self.pause_btn.grid(row=0, column=1, sticky=tk.EW, padx=2, pady=(3, 6)) + self.end_btn = ttk.Button(btns, text="结束", command=self.on_end_flow) + self.end_btn.grid(row=0, column=2, sticky=tk.EW, padx=(2, 6), pady=(3, 6)) - info = ttk.LabelFrame(self.root, text="状态") - info.pack(fill=tk.X, **pad) - ttk.Label(info, textvariable=self.flow_var, wraplength=800, justify=tk.LEFT).pack( - anchor=tk.W, padx=6, pady=2 + self.desktop_options = ttk.LabelFrame(self.root, text="桌面拾取选项") + self.under_cursor_chk = ttk.Checkbutton( + self.desktop_options, + text="光标下优先（悬停、不点击）", + variable=self.under_cursor_var, ) - ttk.Label(info, textvariable=self.target_var, wraplength=800, justify=tk.LEFT).pack( - anchor=tk.W, padx=6, pady=2 + self.under_cursor_chk.pack(anchor=tk.W, padx=8, pady=(2, 0)) + self.wecom_chk = ttk.Checkbutton( + self.desktop_options, + text="企微搜索浮层优先（SearchResultWindow2）", + variable=self.prefer_wecom_search_var, ) - ttk.Label(info, textvariable=self.status_var, wraplength=800, justify=tk.LEFT).pack( - anchor=tk.W, padx=6, pady=2 + self.wecom_chk.pack(anchor=tk.W, padx=8, pady=(0, 4)) + + self.result_panel = ttk.LabelFrame(self.root, text="运行状态与拾取结果") + self.result_panel.pack(fill=tk.BOTH, expand=True, padx=8, pady=(4, 8)) + + self.status_text = tk.Text( + self.result_panel, + height=3, + wrap=tk.NONE, + borderwidth=0, + padx=5, + pady=3, + ) + self.status_text.pack(fill=tk.X, padx=5, pady=(3, 2)) + self.status_text.configure(state=tk.DISABLED) + ttk.Separator(self.result_panel, orient=tk.HORIZONTAL).pack( + fill=tk.X, padx=5, pady=(0, 3) ) - tree_box = ttk.LabelFrame(self.root, text="本步拾取结果") - tree_box.pack(fill=tk.BOTH, expand=True, **pad) - scroll = ttk.Scrollbar(tree_box) + detail_box = ttk.Frame(self.result_panel) + detail_box.pack(fill=tk.BOTH, expand=True, padx=3, pady=(0, 3)) + scroll = ttk.Scrollbar(detail_box) scroll.pack(side=tk.RIGHT, fill=tk.Y) self.detail = tk.Text( - tree_box, - height=24, + detail_box, + height=10, wrap=tk.WORD, + borderwidth=0, yscrollcommand=scroll.set, ) - self.detail.pack(fill=tk.BOTH, expand=True, padx=6, pady=6) + self.detail.pack(fill=tk.BOTH, expand=True, padx=(3, 0), pady=3) scroll.config(command=self.detail.yview) self.detail.insert( tk.END, - "1) 填写流程名，点「开始流程」——新建一份流程文档\n" - "2) 默认已勾选「自动拾取」：先 5 秒倒数（请点目标窗到最前端），\n" - " 然后串行：dump → 保存 → 等 2 秒 → 再 dump … 直到「结束流程」\n" - " 本窗口保持置顶方便看进度；选目标看焦点并排除自身\n" - " 不强制最大化目标窗（保持你当时的大小/位置）\n" - "3) 也可取消自动，改为手动点「拾取当前页」\n" - "4) 点「结束流程」——退出循环并关闭文档\n\n" - "完整内容在 _captures/<流程>/flow.json、SUMMARY.md、steps/\n", + "1. 选择拾取模式\n" + "2. 点击「开始流程」，然后操作目标窗口\n" + "3. 完成后点击「结束流程」\n\n" + "动作、元素定位和诊断结果会显示在这里。\n", ) self.detail.configure(state=tk.DISABLED) - - tip = ( - "一个流程 = 一份文档；多次拾取 = 多个 steps。" - "自动拾取严格串行：上一拍完全结束才会开始下一拍。" - "给 AI 时优先丢 SUMMARY.md + 各步 locatable.json。" - ) - ttk.Label(self.root, text=tip, wraplength=800, foreground="#444").pack( - fill=tk.X, padx=8, pady=6 - ) self._refresh_button_states() def _flow_is_open(self) -> bool: @@ -168,24 +580,94 @@ class InspectorApp: def _in_start_countdown(self) -> bool: return self._countdown_after_id is not None or self._countdown_left > 0 + def _on_recording_mode_change(self) -> None: + self._persist_prefs() + mode = self._current_recording_mode() + if mode == _MODE_LEGACY: + self._set_status("已选 Legacy 间隔 dump：将按约 2 秒串行写 steps（兼容旧行为）。") + elif mode == _MODE_GUIDED: + self._set_status("已选引导录制：每捕获一个动作后在本框展示定位与后置条件建议。") + elif mode == _MODE_CHECKPOINT: + self._set_status("已选手动检查点：请用「采集界面检查点」；不会自动录用户操作。") + self.auto_pick_var.set(False) + else: + self._set_status("已选自动录制：Recorder 捕获用户操作并双写 v1/v2 资产。") + def _refresh_button_states(self) -> None: open_flow = self._flow_is_open() counting = self._in_start_countdown() - # Manual pick only when open, idle, and not in auto loop / countdown - can_manual = ( - open_flow - and not self._busy - and not self._auto_loop_active - and not counting - ) - self.pick_btn.configure(state=tk.NORMAL if can_manual else tk.DISABLED) - # End must stay clickable during a long dump so the loop can be stopped self.end_btn.configure(state=tk.NORMAL if open_flow else tk.DISABLED) self.start_btn.configure( - state=tk.DISABLED if (self._busy or counting) else tk.NORMAL + state=tk.DISABLED if (self._busy or counting or open_flow) else tk.NORMAL ) - # Don't flip checkbox mid-dump in a confusing way; allow toggle to stop loop - self.auto_chk.configure(state=tk.NORMAL) + try: + self.pause_btn.configure( + state=( + tk.NORMAL + if (open_flow and self._session and not self._busy and not counting) + else tk.DISABLED + ) + ) + except Exception: + pass + + def _update_rec_stats(self) -> None: + if not self._session: + self.rec_stats_var.set( + "recorder: idle | actions=0 states=0 observations=0 elements=0" + ) + return + st = self._session.stats() + input_health = st.get("input_backend") or {} + input_summary = "" + if input_health: + input_summary = ( + f" input={input_health.get('backend', 'unknown')}" + f" raw={input_health.get('raw_input_events', 0)}" + f" mouse={input_health.get('mouse_events', 0)}" + f" keyboard={input_health.get('keyboard_events', 0)}" + f" input_dropped={input_health.get('queue_dropped', 0)}" + f" input_error={input_health.get('last_error') or '-'} |" + ) + self.rec_stats_var.set( + f"recorder={st.get('recorder_status')}/{st.get('surface_recorder')} | " + f"actions={st.get('action_count')} states={st.get('state_count')} " + f"observations={st.get('observation_count')} elements={st.get('element_count')} " + f"queue={st.get('queued_events', 0)} dropped={st.get('dropped_events', 0)} | " + f"{input_summary} " + f"last={st.get('last_action_type') or '-'} " + f"target={st.get('last_target') or '-'} " + f"page={st.get('last_page_id') or '-'} " + f"obs={st.get('last_observation_id') or '-'} " + f"match={st.get('last_match_count')} unique={st.get('last_unique')}" + ) + + def _update_flow_label(self) -> None: + if not self.flow_dir or not self.flow: + self.flow_var.set("流程：未开始（请先点「开始流程」）") + return + if self._auto_loop_active: + auto = "LOOP" if self._is_legacy_interval() else "REC" + elif self.auto_pick_var.get() and self._flow_is_open(): + auto = "armed" + else: + auto = "off" + v2 = (self.flow or {}).get("v2") or {} + live = self._session.stats() if self._session else {} + action_count = live.get("action_count", v2.get("action_count", 0)) + state_count = live.get("state_count", v2.get("state_count", 0)) + observation_count = live.get( + "observation_count", v2.get("observation_count", 0) + ) + self.flow_var.set( + f"流程：{self.flow.get('flow_name')} | mode={self._current_mode()} | " + f"rec={self._current_recording_mode()} | " + f"status={self.flow.get('status')} | " + f"legacy_steps={self.flow.get('step_count')} " + f"actions={action_count} states={state_count} " + f"observations={observation_count} | auto={auto} | {self.flow_dir}" + ) + self._update_rec_stats() def _set_status(self, text: str) -> None: self.status_var.set(text) @@ -197,11 +679,63 @@ class InspectorApp: self.detail.see("1.0") self.detail.configure(state=tk.DISABLED) + def _on_session_action_update(self, action, phase: str) -> None: + """Snapshot an ActionRecord on a worker, then hand it safely to Tk.""" + target_snapshot = dict(getattr(action, "target_snapshot", {}) or {}) + evidence = dict(getattr(action, "evidence", {}) or {}) + input_payload = dict(getattr(action, "input", {}) or {}) + scope = dict(target_snapshot.get("window_scope") or {}) + input_summary = "" + if input_payload.get("key"): + input_summary = str(input_payload.get("key")) + elif input_payload.get("text"): + value = str(input_payload.get("text")) + input_summary = value if len(value) <= 80 else value[:77] + "..." + item = { + "id": str(getattr(action, "id", "") or ""), + "sequence": int(getattr(action, "sequence", 0) or 0), + "action_type": str(getattr(action, "action_type", "") or ""), + "surface": str(getattr(action, "surface", "") or ""), + "phase": phase, + "target_name": target_snapshot.get("name") + or target_snapshot.get("automation_id") + or target_snapshot.get("semantic_name") + or target_snapshot.get("role") + or target_snapshot.get("tag") + or "-", + "target_type": target_snapshot.get("control_type") + or target_snapshot.get("role") + or target_snapshot.get("tag") + or "-", + "window_title": scope.get("title") or "", + "window_class": scope.get("class_name") or "", + "input_summary": input_summary, + "locator": evidence.get("primary_locator") or "", + "match_count": evidence.get("locator_match_count"), + "unique": evidence.get("locator_unique"), + "error": evidence.get("eventual_evidence_error") + or evidence.get("error") + or "", + } + self._post_ui(self._apply_live_action_update, item) + + def _apply_live_action_update(self, item: dict) -> None: + updated = _merge_current_live_action(self._current_live_action, item) + if updated is self._current_live_action: + return + self._current_live_action = updated + assert updated is not None + self.target_var.set( + f"当前动作：#{updated.get('sequence')} {updated.get('action_type')} · " + f"{updated.get('phase')}" + ) + self._set_detail("当前步骤\n\n" + _format_live_action(updated)) + def _show_picking_placeholder(self, *, auto: bool) -> None: n = int((self.flow or {}).get("step_count") or 0) + 1 - mode = "自动" if auto else "手动" - self.target_var.set(f"{mode}拾取中…（即将写入第 {n} 步）") - self._set_status(f"正在{mode}拾取：已清空本框上次结果，请稍候…") + mode = "自动" if auto else "检查点" + self.target_var.set(f"{mode}采集中…（即将写入第 {n} 步）") + self._set_status(f"正在{mode}采集：已清空本框上次结果，请稍候…") self._set_detail( "（已清空上次本步结果）\n\n" f"正在读取控件树，完成后写入当前流程第 {n} 步…\n" @@ -213,21 +747,6 @@ class InspectorApp: except Exception: return set() - def _update_flow_label(self) -> None: - if not self.flow_dir or not self.flow: - self.flow_var.set("流程：未开始（请先点「开始流程」）") - return - if self._auto_loop_active: - auto = "LOOP" - elif self.auto_pick_var.get() and self._flow_is_open(): - auto = "armed" - else: - auto = "off" - self.flow_var.set( - f"流程：{self.flow.get('flow_name')} | status={self.flow.get('status')} | " - f"steps={self.flow.get('step_count')} | auto={auto} | {self.flow_dir}" - ) - def _cancel_auto_timer(self) -> None: if self._auto_after_id is not None: try: @@ -245,17 +764,265 @@ class InspectorApp: self._countdown_after_id = None self._countdown_left = 0 + def _cancel_heartbeat(self) -> None: + if self._heartbeat_after_id is not None: + try: + self.root.after_cancel(self._heartbeat_after_id) + except Exception: + pass + self._heartbeat_after_id = None + def _stop_auto_loop(self) -> None: self._auto_loop_active = False self._cancel_auto_timer() self._cancel_countdown() + self._cancel_heartbeat() - def _start_auto_loop(self, *, first_delay_ms: int = _AUTO_START_DELAY_MS) -> None: + def _stop_session(self) -> None: + if self._session is not None: + try: + self._session.stop() + except Exception: + pass + self._session = None + self._rec_paused = False + try: + self.pause_btn.configure(text="暂停") + except Exception: + pass + + def _schedule_heartbeat(self) -> None: + self._cancel_heartbeat() + if not self._auto_loop_active or not self._flow_is_open(): + return + if self._is_legacy_interval(): + return + self._heartbeat_after_id = self.root.after(_HEARTBEAT_MS, self._heartbeat_tick) + + def _heartbeat_tick(self) -> None: + self._heartbeat_after_id = None + if not self._auto_loop_active or not self._flow_is_open(): + return + if self._session and not self._rec_paused: + try: + self._session.heartbeat() + except Exception: + pass + self._update_flow_label() + # Guided mode: refresh last action detail + if self._current_recording_mode() == _MODE_GUIDED: + self._show_guided_last_action() + self._schedule_heartbeat() + + def _show_guided_last_action(self) -> None: + if not self._session: + return + act = self._session.manager.last_action + if not act: + return + lines = [ + f"最近动作 #{act.sequence} {act.action_type} status={act.status}", + f"target: {(act.target_snapshot or {}).get('name') or (act.target_snapshot or {}).get('automation_id')}", + f"scope: {(act.target_snapshot or {}).get('window_scope')}", + f"primary locator: {(act.evidence or {}).get('primary_locator')}", + f"match count: {(act.evidence or {}).get('locator_match_count')} unique={(act.evidence or {}).get('locator_unique')}", + "proposed postconditions:", + ] + for p in (act.proposed_postconditions or [])[:8]: + lines.append(f" - [{p.get('status')}] {p.get('description')}") + if self._session.manager.last_warning: + lines.append(f"warning: {self._session.manager.last_warning}") + self._set_detail("\n".join(lines)) + + def _make_dump_fn(self): + """Build a dump callable for SnapshotService (runs on worker threads).""" + + # Snapshot every Tk-owned value on the main thread. The returned + # callable is deliberately free of Tk calls so it cannot participate + # in a Tk <-> recorder lock inversion. + mode = self._current_mode() + under_cursor = bool(self.under_cursor_var.get()) + prefer_search = bool(self.prefer_wecom_search_var.get()) + sticky_pid = self.sticky_pid + exclude_hwnds = self._exclude_hwnds() + max_depth = self.max_depth + max_nodes = self.max_nodes + warmed_pids = self._warmed_pids + + def dump_fn( + *, page_id: str = "", document_id: str = "", frame_id: str = "", + target_hwnd: int = 0, target_pid: int = 0, + target_class_name: str = "", target_title: str = "", + ): + uia.ensure_com_sta() + surface = mode + sel_meta: dict = {"pick_mode": mode, "sticky_pid": sticky_pid} + if surface == target.SURFACE_BROWSER: + info = self._synthetic_browser_info() + sel_meta["strategy"] = "playwright_session" + elif target_hwnd: + try: + info = uia.window_info_from_hwnd(int(target_hwnd)) + except Exception: + info = None + if info is None and target_class_name == "SearchResultWindow2": + info = uia.find_wecom_search_result_window( + pid=int(target_pid or 0) or None, + exclude_hwnds=exclude_hwnds, + ) + if info is None: + raise RuntimeError( + f"event window disappeared: hwnd={target_hwnd} class={target_class_name}" + ) + cands = [info] + sel_meta.update( + { + "strategy": "event_window", + "chosen_hwnd": int(info.hwnd), + "chosen_class": info.class_name, + "event_title": target_title, + } + ) + sel_meta["candidate_count"] = 1 + else: + info, cands, win_meta = uia.resolve_capture_target( + exclude_hwnds=exclude_hwnds, + prefer_pid=sticky_pid, + under_cursor=under_cursor, + prefer_wecom_search_popup=prefer_search, + ) + sel_meta.update(win_meta) + sel_meta["candidate_count"] = len(cands) + driver = get_driver(surface) + warm_key = ( + (int(info.pid), str(info.class_name or "")) + if surface == target.SURFACE_WINDOWS + else int(info.pid or 0) + ) + warmup_mode = "light" if warm_key in warmed_pids else "full" + if surface == target.SURFACE_WINDOWS: + # Recording already has a persisted event-time element atlas. + # Evidence may probe the full tree, but must never run a + # blocking product wake in the middle of user input. + warmup_mode = "probe" + if target_class_name == "SearchResultWindow2": + # A search popup may live for only a few seconds. Persist its + # bounded tree immediately; heavyweight kit wake/FindAll remains + # supplementary and must not make the event-time snapshot miss it. + warmup_mode = "off" + # Recorder path: after-action dumps may warm; click capture itself never warms. + dump_kwargs = dict( + info=info, + selection_meta=sel_meta, + max_depth=(12 if target_class_name == "SearchResultWindow2" else max_depth), + max_nodes=(1500 if target_class_name == "SearchResultWindow2" else max_nodes), + warmup_mode=warmup_mode, + warmed_pids=warmed_pids, + ) + if surface == target.SURFACE_BROWSER: + dump_kwargs.update( + page_id=page_id, + document_id=document_id, + frame_id=frame_id, + ) + result = driver.dump(**dump_kwargs) + if info.pid: + warmed_pids.add(warm_key) + return result + + return dump_fn + + def _start_recording_session( + self, *, entry_dump=None, checkpoint_only: bool = False + ) -> None: + if not self.flow_dir or not self.flow: + return + rec_mode = _MODE_CHECKPOINT if checkpoint_only else self._current_recording_mode() + session = FlowSession( + self.flow_dir, + flow_name=str(self.flow.get("flow_name") or "flow"), + surface=self._current_mode(), + recording_mode=rec_mode, + dump_fn=self._make_dump_fn(), + exclude_hwnds=self._exclude_hwnds, + on_action_update=self._on_session_action_update, + stability_timeout_sec=_STABILITY_TIMEOUT_SEC, + ) + self._session = session + if rec_mode == _MODE_LEGACY: + self._auto_loop_active = True + self._schedule_auto_next(_AUTO_START_DELAY_MS) + return + + self._busy = True + self._refresh_button_states() + self._set_status("正在后台采集入口状态，界面仍可响应……") + + def work() -> None: + try: + if rec_mode == _MODE_CHECKPOINT: + session.manager.action_flow.recording_mode = _MODE_CHECKPOINT + session.manager.start(entry_dump=entry_dump) + session._refresh_stats() + else: + session.start(entry_dump=entry_dump) + stats = session.stats() + if int(stats.get("state_count") or 0) == 0: + warning = str(stats.get("warning") or "入口状态采集没有产物") + raise RuntimeError(f"ERROR:ENTRY_CAPTURE_FAILED {warning}") + except Exception as exc: + self._post_ui(self._on_recording_session_started, session, rec_mode, exc) + try: + session.stop() + except Exception: + pass + return + self._post_ui(self._on_recording_session_started, session, rec_mode, None) + + threading.Thread( + target=work, name="ui-inspector-session-start", daemon=True + ).start() + + def _on_recording_session_started( + self, session: FlowSession, rec_mode: str, error: BaseException | None + ) -> None: + if self._session is not session: + return + self._busy = False + if error is not None: + self._session = None + self._set_status(f"录制会话启动失败: {error}") + self._refresh_button_states() + return self._auto_loop_active = True - self._pending_end = False + self._rec_paused = False self._update_flow_label() self._refresh_button_states() - self._schedule_auto_next(first_delay_ms) + if rec_mode == _MODE_CHECKPOINT: + self._auto_loop_active = False + self._set_status("入口状态已采集；可手动添加界面检查点。") + else: + self.target_var.set("动作录制中：等待用户操作") + self._set_status("入口状态已采集，动作录制已就绪。") + self._schedule_heartbeat() + + def _start_auto_loop(self, *, first_delay_ms: int = _AUTO_START_DELAY_MS) -> None: + """Legacy interval loop OR action-driven recording start.""" + self._pending_end = False + if self._is_legacy_interval(): + self._auto_loop_active = True + self._update_flow_label() + self._refresh_button_states() + self._schedule_auto_next(first_delay_ms) + return + # Action-driven: the desktop baseline is intentionally lightweight; + # full provider evidence is collected asynchronously after actions. + entry_dump = ( + self._desktop_entry_dump + if self._current_mode() == target.SURFACE_WINDOWS + else None + ) + self._start_recording_session(entry_dump=entry_dump) def _set_inspector_topmost(self, enabled: bool) -> None: try: @@ -272,7 +1039,7 @@ class InspectorApp: pass def _arm_auto_with_countdown(self, *, seconds: int = _START_COUNTDOWN_SEC) -> None: - """Visible countdown, then resolve frontmost and enter auto loop.""" + """Visible countdown, then enter mode-specific auto loop.""" if not self._flow_is_open(): return # Stop prior loop/timers without clearing the countdown we are about to arm. @@ -284,18 +1051,41 @@ class InspectorApp: self._set_inspector_topmost(True) self._countdown_left = max(1, int(seconds)) self._update_flow_label() - self._set_status( - f"倒计时 {self._countdown_left} 秒：请把要采集的窗口点到最前端" - "（本面板置顶看进度；不强制放大目标窗）。" - ) - self._set_detail( - f"流程：{self.flow_dir}\n\n" - f"- {self._countdown_left} 秒内请点击目标窗口（给它焦点）\n" - f"- 本 Inspector 保持置顶，方便看进度；选窗排除自身\n" - f"- 倒计时结束后：识别最前端 → dump → 等 2 秒 → 再 dump …\n" - f"- 目标窗保持你当前的大小/位置（不强制最大化）\n" - f"- 可点「结束流程」或取消「自动拾取」中止\n" - ) + mode = self._current_mode() + if mode == target.SURFACE_BROWSER: + url = (self.start_url_var.get() or "").strip() + behavior = ( + "Legacy：打开浏览器后每 2 秒采集一次" + if self._is_legacy_interval() + else "打开浏览器后监听用户动作，并采集动作后的稳定状态" + ) + self._set_status( + f"倒计时 {self._countdown_left} 秒：将打开浏览器并进入 {url or '(起始 URL)'}" + ) + self._set_detail( + f"流程：{self.flow_dir}\n\n" + f"- 模式：浏览器自动化\n" + f"- 起始 URL：{url or '(空，自行在浏览器中打开目标站)'}\n" + f"- 倒计时结束后：{behavior}\n" + f"- 可点「结束流程」或取消「自动拾取」中止\n" + ) + else: + behavior = ( + "Legacy：识别窗口后每 2 秒采集一次" + if self._is_legacy_interval() + else "识别窗口后监听用户动作，并采集动作后的稳定状态" + ) + self._set_status( + f"倒计时 {self._countdown_left} 秒：请把要采集的桌面窗口点到最前端" + "（本面板置顶看进度；不强制放大目标窗）。" + ) + self._set_detail( + f"流程：{self.flow_dir}\n\n" + f"- 模式：桌面自动化\n" + f"- {self._countdown_left} 秒内请点击目标窗口（给它焦点）\n" + f"- 倒计时结束后：{behavior}\n" + f"- 可点「结束流程」或取消「自动拾取」中止\n" + ) self._refresh_button_states() self._countdown_after_id = self.root.after(1000, self._countdown_tick) @@ -312,19 +1102,101 @@ class InspectorApp: return self._countdown_left -= 1 if self._countdown_left > 0: - self._set_status( - f"倒计时 {self._countdown_left} 秒：请把要采集的窗口点到最前端…" - ) + mode = self._current_mode() + if mode == target.SURFACE_BROWSER: + self._set_status( + f"倒计时 {self._countdown_left} 秒：准备打开浏览器…" + ) + else: + self._set_status( + f"倒计时 {self._countdown_left} 秒：请把要采集的窗口点到最前端…" + ) self._refresh_button_states() self._countdown_after_id = self.root.after(1000, self._countdown_tick) return self._countdown_left = 0 self._refresh_button_states() - self._set_status("倒计时结束，正在识别最前端窗口…") - self._begin_auto_with_probe() + if self._current_mode() == target.SURFACE_BROWSER: + self._set_status("倒计时结束，正在打开浏览器…") + self._begin_auto_browser() + else: + self._set_status("倒计时结束，正在识别最前端窗口…") + self._begin_auto_with_probe() + + def _begin_auto_browser(self) -> None: + """Launch Playwright with start URL, then enter auto loop (no Win32 probe).""" + if self._busy: + return + self._busy = True + self._auto_loop_active = False + self._refresh_button_states() + self._set_inspector_topmost(True) + self._ensure_inspector_visible() + url = (self.start_url_var.get() or "").strip() + status_url = url or "(未填起始 URL，打开空白页后可自行导航)" + self._set_status(f"浏览器模式：正在打开 {status_url} …") + self._set_detail( + f"启动 Playwright…\n\n" + f"- 起始 URL：{url or '(空)'}\n" + f"- profile：{kit_browser.profile_dir()}\n" + f"- 打开完成后进入{'Legacy 间隔采集' if self._is_legacy_interval() else '动作录制'}\n" + ) + + def work() -> None: + try: + # 有 URL 才 force_goto；空则只启浏览器 + sess = kit_browser.ensure_session( + start_url=url or "about:blank", + force_goto=bool(url), + ) + self._post_ui(self._on_browser_ready, sess, url) + except Exception as exc: + self._post_ui(self._on_frontmost_fail, exc) + + threading.Thread(target=work, daemon=True).start() + + def _on_browser_ready(self, sess: dict, url: str) -> None: + self._busy = False + self._refresh_button_states() + live = str(sess.get("url") or "about:blank") + if url and (not live or live == "about:blank" or live.startswith("about:")): + self._on_frontmost_fail( + RuntimeError( + f"ERROR:NAVIGATION_FAILED requested={url!r} actual={live!r}" + ) + ) + return + navigation = dict(sess.get("navigation") or {}) + nav_warning = str(navigation.get("warning") or "") + self._set_status( + f"浏览器已就绪：{live} — " + + ("开始 Legacy 间隔采集。" if self._is_legacy_interval() else "正在采集入口状态。") + ) + self._set_detail( + f"采集目标（Playwright）:\n" + f" mode=browser\n" + f" start_url={url or '(空)'}\n" + f" live_url={live}\n" + f" profile={sess.get('profile_dir')}\n" + f" reused={sess.get('reused')}\n\n" + + (f"导航警告：{nav_warning}\n\n" if nav_warning else "") + + ( + "即将进入 Legacy 自动循环（步间 2 秒）…\n" + if self._is_legacy_interval() + else "即将启动动作录制器…\n" + ) + + ( + "" + if url + else "未填起始 URL：请在弹出的浏览器中打开目标站（空页会自动跳过写入）。\n" + ) + ) + self._set_inspector_topmost(True) + self._ensure_inspector_visible() + self._start_auto_loop(first_delay_ms=_AUTO_START_DELAY_MS) def _begin_auto_with_probe(self) -> None: - """Resolve frontmost target (not Inspector), warmup best-effort, then loop.""" + """Desktop mode: resolve frontmost target, warmup best-effort, then loop.""" if self._busy: return self._busy = True @@ -334,28 +1206,26 @@ class InspectorApp: self._set_inspector_topmost(True) self._ensure_inspector_visible() self._set_status( - "自动拾取：正在识别最前端窗口（焦点窗=采集目标，已排除本面板）…" + "桌面模式：正在识别最前端窗口（焦点窗=采集目标，已排除本面板）…" ) self._set_detail( "识别最前端窗口中…\n\n" + "- 模式：桌面自动化（强制 UIA）\n" "- 规则：谁有焦点就采谁（与窗口大小无关）\n" - "- 本面板可置顶；选目标时排除 Inspector 自身\n" - "- 不强制最大化目标窗\n" - "- 若是企微：自动调用 kit ensure_ui_ready（与技能同路径）\n" - "- FindAll 多少只作提示，不会因此改采别的软件\n" + "- 若是企微：自动调用 kit ensure_ui_ready\n" ) + exclude_hwnds = self._exclude_hwnds() def work() -> None: try: uia.ensure_com_sta() report = uia.prepare_frontmost_for_capture( - exclude_hwnds=self._exclude_hwnds(), + exclude_hwnds=exclude_hwnds, timeout_sec=_FRONTMOST_WAIT_SEC, ) - self.root.after(0, lambda r=report: self._on_frontmost_ready(r)) + self._post_ui(self._on_frontmost_ready, report) except Exception as exc: - err = exc - self.root.after(0, lambda e=err: self._on_frontmost_fail(e)) + self._post_ui(self._on_frontmost_fail, exc) threading.Thread(target=work, daemon=True).start() @@ -365,7 +1235,38 @@ class InspectorApp: win = report.get("window") or {} if win.get("pid"): self.sticky_pid = int(win["pid"]) - self._warmed_pids.add(int(win["pid"])) + bounds = dict(win.get("rect") or win.get("bounds") or {}) + root_row = { + "name": str(win.get("title") or ""), + "value": "", + "automation_id": "", + "control_type": "WindowControl", + "class_name": str(win.get("class_name") or ""), + "framework_id": "Win32", + "is_enabled": True, + "is_offscreen": False, + "bounds": bounds, + "window_scope": dict(win), + } + self._desktop_entry_dump = DumpResult( + surface=target.SURFACE_WINDOWS, + driver="uiautomation-light", + window=dict(win), + tree={**root_row, "children": []}, + index=[dict(root_row)], + locatable=[dict(root_row)], + stats={ + "capture_kind": "entry_light", + "nodes_kept": 1, + "locatable_count": 1, + "provider_enrichment": "deferred", + }, + selection={ + "strategy": str(report.get("strategy") or "foreground_only"), + "entry_light": True, + }, + use_hwnd_screenshot=False, + ) import kit_wake kit = report.get("kit_wake") or {} @@ -387,18 +1288,25 @@ class InspectorApp: f"（UIA 仍偏稀 FindAll={report.get('findall')}；仍按此前端窗采集）" ) self._set_status( - f"已锁定最前端：{win.get('title')!r} / {win.get('class_name')!r} " - f"findall={report.get('findall')} — 开始自动串行拾取。{sparse_note}" + f"已锁定桌面窗：{win.get('title')!r} " + f"findall={report.get('findall')} — " + + ("开始 Legacy 间隔采集。" if self._is_legacy_interval() else "正在采集入口状态。") + + sparse_note ) self._set_detail( - f"采集目标（最前端）:\n" + f"采集目标（桌面 UIA）:\n" + f" mode=windows\n" f" title={win.get('title')!r}\n" f" class={win.get('class_name')!r}\n" f" hwnd={win.get('hwnd')} pid={win.get('pid')}\n" f" findall={report.get('findall')} sparse={report.get('uia_sparse')} " f"cold={report.get('provider_cold')}\n" f" {kit_line}\n\n" - "即将进入自动循环（步间 2 秒）…\n" + + ( + "即将进入 Legacy 自动循环（步间 2 秒）…\n" + if self._is_legacy_interval() + else "即将启动动作录制器…\n" + ) ) self._set_inspector_topmost(True) self._ensure_inspector_visible() @@ -460,9 +1368,9 @@ class InspectorApp: self._update_flow_label() self._refresh_button_states() if was and self._flow_is_open(): - self._set_status("已停止自动循环/倒计时；仍可手动点「拾取当前页」。") + self._set_status("已停止自动录制/倒计时；仍可手动点「采集界面检查点」。") elif self._flow_is_open(): - self._set_status("自动拾取已关闭；可手动点「拾取当前页」。") + self._set_status("自动录制已关闭；可手动点「采集界面检查点」。") def on_open_out(self) -> None: path = self.flow_dir if self.flow_dir else self.captures_root @@ -475,6 +1383,15 @@ class InspectorApp: def on_start_flow(self) -> None: if self._busy: return + mode = self._current_mode() + if mode == target.SURFACE_ANDROID: + messagebox.showwarning( + "UI Inspector", + "手机自动化尚未实现。\n请先选择「浏览器自动化」或「桌面自动化」。", + ) + return + self._persist_prefs() + if self.flow and self.flow.get("status") == "open": if not messagebox.askyesno( "UI Inspector", @@ -487,34 +1404,74 @@ class InspectorApp: art.end_flow(self.flow_dir, self.flow) # type: ignore[arg-type] except Exception: pass + self._stop_session() name = (self.flow_name_var.get() or "").strip() if not name: - name = simpledialog.askstring("流程名", "请输入流程名称：", initialvalue="desktop-flow") + default_name = _flow_name_for_mode(mode) + name = simpledialog.askstring( + "流程名", "请输入流程名称：", initialvalue=default_name + ) if not name: return self.flow_name_var.set(name) try: - self.flow_dir = art.create_flow(self.captures_root, flow_name=name) + self.flow_dir = art.create_flow( + self.captures_root, + flow_name=name, + recording_mode=self._current_recording_mode(), + ) self.flow = art.load_flow(self.flow_dir) self._pending_end = False self._warmed_pids.clear() + self._current_live_action = None + self._desktop_entry_dump = None + self._stop_session() self._update_flow_label() self._refresh_button_states() - if self.auto_pick_var.get(): + if self._current_recording_mode() == _MODE_CHECKPOINT: + self.auto_pick_var.set(False) + if self.auto_pick_var.get() and self._current_recording_mode() != _MODE_CHECKPOINT: # No modal / no minimize: keep panel visible for countdown + progress. self._arm_auto_with_countdown(seconds=_START_COUNTDOWN_SEC) else: self._stop_auto_loop() self._ensure_inspector_visible() - self._set_status( - f"已开始流程：{self.flow_dir.name}。请翻到目标页后点「拾取当前页」。" - ) + if mode == target.SURFACE_BROWSER: + url = (self.start_url_var.get() or "").strip() + self._set_status( + f"已开始流程：{self.flow_dir.name}。" + + ( + f"将打开 {url}；请点「采集界面检查点」。" + if url + else "已启浏览器（无起始 URL）；请自行打开目标站后点「采集界面检查点」。" + ) + ) + # Eager open browser so manual pick is ready + def _open() -> None: + try: + kit_browser.ensure_session( + start_url=url or "about:blank", + force_goto=bool(url), + ) + except Exception: + pass + + threading.Thread(target=_open, daemon=True).start() + else: + self._set_status( + f"已开始流程：{self.flow_dir.name}。请翻到目标页后点「采集界面检查点」。" + ) + # Manual mode owns no surface recorder; it only keeps the v2 + # manager available for explicit checkpoints. + self._start_recording_session(checkpoint_only=True) self._set_detail( f"流程已创建：{self.flow_dir}\n\n" - f"- flow.json / SUMMARY.md / steps/\n" - f"- 手动点「拾取当前页」追加一步；或勾选「自动拾取」进入串行循环。\n" + f"- 模式：{mode}\n" + f"- 录制：{self._current_recording_mode()}\n" + f"- flow.json / action_flow.json / element_catalog.json / steps/\n" + f"- 手动点「采集界面检查点」；或勾选自动录制用户操作。\n" ) except Exception as exc: messagebox.showerror("UI Inspector", str(exc)) @@ -525,21 +1482,80 @@ class InspectorApp: if self.flow.get("status") != "open": self._refresh_button_states() return + session = self._session + flow_dir = self.flow_dir + flow = self.flow + self._session = None + self._busy = True + self._refresh_button_states() + self._set_status("正在后台刷新动作队列并结束流程……") + + def work() -> None: + try: + if session is not None: + session.stop() + latest_flow = art.load_flow(flow_dir) + ended = art.end_flow(flow_dir, latest_flow) + self._post_ui(self._on_end_flow_finished, ended, None) + except Exception as exc: + self._post_ui(self._on_end_flow_finished, None, exc) + + threading.Thread( + target=work, name="ui-inspector-session-stop", daemon=True + ).start() + + def _on_end_flow_finished( + self, ended_flow: dict | None, error: BaseException | None + ) -> None: + self._busy = False + if error is not None or ended_flow is None: + self._refresh_button_states() + messagebox.showerror("UI Inspector", str(error or "结束流程失败")) + return try: - self.flow = art.end_flow(self.flow_dir, self.flow) + self.flow = ended_flow self._update_flow_label() self._refresh_button_states() n = self.flow.get("step_count") or 0 - self._set_status(f"流程已结束，共 {n} 步。可再点「开始流程」开新文档。") + v2 = self.flow.get("v2") or {} + self._set_status( + f"流程已结束：actions={v2.get('action_count', '?')}，" + f"states={v2.get('state_count', '?')}，" + f"observations={v2.get('observation_count', '?')}，" + f"legacy_steps={n}。" + "可再点「开始流程」开新文档。" + ) self._set_detail( f"流程已结束：{self.flow_dir}\n\n" - f"- 共 {n} 步\n" - f"- 请打开 SUMMARY.md 查看总览\n" + f"- v1 steps: {n}\n" + f"- v2 actions: {v2.get('action_count')}\n" + f"- 请打开 SUMMARY.md / action_flow.json / element_catalog.json\n" + ) + messagebox.showinfo( + "UI Inspector", + f"流程已结束\nactions={v2.get('action_count')} " + f"states={v2.get('state_count')} " + f"observations={v2.get('observation_count')} " + f"legacy_steps={n}\n{self.flow_dir}", ) - messagebox.showinfo("UI Inspector", f"流程已结束\n共 {n} 步\n{self.flow_dir}") except Exception as exc: messagebox.showerror("UI Inspector", str(exc)) + def on_pause_resume(self) -> None: + if not self._session or not self._flow_is_open(): + return + if self._rec_paused: + self._session.resume() + self._rec_paused = False + self.pause_btn.configure(text="暂停") + self._set_status("已继续录制。") + else: + self._session.pause() + self._rec_paused = True + self.pause_btn.configure(text="继续") + self._set_status("已暂停录制（Inspector 自身操作不会写入）。") + self._update_rec_stats() + def on_end_flow(self) -> None: if not self.flow_dir or not self.flow: messagebox.showwarning("UI Inspector", "没有进行中的流程。") @@ -549,7 +1565,7 @@ class InspectorApp: return self._stop_auto_loop() - # Keep「自动拾取」勾选状态（默认开），方便下一轮直接开始。 + # Keep「自动录制」勾选状态（默认开），方便下一轮直接开始。 if self._busy: self._pending_end = True @@ -561,6 +1577,7 @@ class InspectorApp: self._finish_end_flow() def on_pick_page(self, *, auto: bool = False) -> None: + """Manual checkpoint or legacy auto dump. Action-driven mode uses Recorder.""" if self._busy: return if not self._flow_is_open(): @@ -569,56 +1586,158 @@ class InspectorApp: return if auto and not self._auto_loop_active: return + # In action-driven mode, auto ticks should not dump steps. + if auto and not self._is_legacy_interval(): + return self._busy = True self._refresh_button_states() self._show_picking_placeholder(auto=auto) self.root.update_idletasks() + # Capture Tk-owned configuration before entering the worker. + mode = self._current_mode() + under_cursor = bool(self.under_cursor_var.get()) + prefer_search = bool(self.prefer_wecom_search_var.get()) + start_url = (self.start_url_var.get() or "").strip() + sticky_pid = self.sticky_pid + exclude_hwnds = self._exclude_hwnds() + flow_dir = self.flow_dir + synthetic_browser_info = ( + self._synthetic_browser_info() + if mode == target.SURFACE_BROWSER + else None + ) + def work() -> None: try: uia.ensure_com_sta() - # Keep panel topmost for progress; selection excludes Inspector hwnd. - self.root.after(0, lambda: self._set_inspector_topmost(True)) - flow = art.load_flow(self.flow_dir) # type: ignore[arg-type] + flow = art.load_flow(flow_dir) # type: ignore[arg-type] if flow.get("status") != "open": raise RuntimeError("流程已结束，取消本步拾取。") - info, cands = uia.find_capture_target( - exclude_hwnds=self._exclude_hwnds(), - prefer_pid=None if auto else self.sticky_pid, + if mode == target.SURFACE_ANDROID: + raise RuntimeError( + "ERROR:SURFACE_UNSUPPORTED 手机自动化尚未实现。" + ) + surface = mode + sel_meta: dict = { + "pick_mode": mode, + "sticky_pid": sticky_pid, + } + if surface == target.SURFACE_BROWSER: + kit_browser.ensure_session( + start_url=start_url or "about:blank", + force_goto=False, + ) + info = synthetic_browser_info + # Best-effort Win32 chrome window for meta only + try: + win_info, cands, win_meta = uia.resolve_capture_target( + exclude_hwnds=exclude_hwnds, + prefer_pid=None, + under_cursor=False, + prefer_wecom_search_popup=False, + ) + if target.surface_from_window_info(win_info) == ( + target.SURFACE_BROWSER + ): + info = win_info + sel_meta.update(win_meta) + sel_meta["candidate_count"] = len(cands) + except Exception: + sel_meta["strategy"] = "playwright_session" + sel_meta["resolved_surface"] = surface + sel_meta["start_url"] = start_url + else: + info, cands, win_meta = uia.resolve_capture_target( + exclude_hwnds=exclude_hwnds, + prefer_pid=None if auto else sticky_pid, + under_cursor=under_cursor, + prefer_wecom_search_popup=prefer_search, + ) + sel_meta.update(win_meta) + sel_meta["candidate_count"] = len(cands) + sel_meta["resolved_surface"] = surface + driver = get_driver(surface) + manual_warm_key = ( + (int(info.pid), str(info.class_name or "")) + if surface == target.SURFACE_WINDOWS + else int(info.pid or 0) ) - # Prefer light after PID already warmed; dump_window_tree escalates - # to full poke if locatable still sparse (still no force maximize). warmup_mode = ( - "light" if int(info.pid) in self._warmed_pids else "full" + "light" if manual_warm_key in self._warmed_pids else "full" ) - tree, index, stats, locatable = uia.dump_window_tree( - info.hwnd, + result = driver.dump( + info=info, + selection_meta=sel_meta, max_depth=self.max_depth, max_nodes=self.max_nodes, - only_visible=False, warmup_mode=warmup_mode, - merge_same_pid_if_sparse=True, + warmed_pids=self._warmed_pids, ) - self._warmed_pids.add(int(info.pid)) - window = info.to_dict() + if info.pid: + self._warmed_pids.add(manual_warm_key) + skip_reason = skip_policy.should_skip_empty_dump( + surface=result.surface, + url=result.url, + locatable=result.locatable, + stats=result.stats, + ) + if skip_reason: + msg = skip_policy.skip_reason_text( + skip_reason, url=result.url + ) + self._post_ui( + self._on_pick_skipped, + msg, + auto=auto, + surface=result.surface, + url=result.url, + ) + return + # Dual-write v2 checkpoint when session exists (manual) + if self._session is not None and not auto: + try: + mgr = self._session.manager + before = mgr.snapshots.latest + after = mgr.snapshots.capture_from_dump( + result, change_reason="checkpoint", force_persist=True + ) + mgr._finalize_action( + action_type="checkpoint", + before=before, + after=after, + target={}, + input_payload={}, + raw_event={"meta": {"source": "manual_checkpoint"}}, + confidence=0.9, + ) + except Exception: + pass + window = result.window + locatable = result.locatable + index = result.index + stats = result.stats label = art.resolve_step_label( flow, - window_title=info.title or "", - window_class=info.class_name or "", + window_title=window.get("title") or info.title or "", + window_class=window.get("class_name") or info.class_name or "", locatable=locatable, ) step_dir, step_idx, step_name = art.add_step_dir( self.flow_dir, # type: ignore[arg-type] flow, - window_title=info.title or "", - window_class=info.class_name or "", + window_title=window.get("title") or info.title or "", + window_class=window.get("class_name") or info.class_name or "", locatable=locatable, label=label, ) meta = art.build_meta( window=window, scene=flow.get("flow_name") or "flow", + surface=result.surface, + driver=result.driver, + url=result.url, extra={ "flow_dir": str(self.flow_dir), "step_index": step_idx, @@ -626,18 +1745,22 @@ class InspectorApp: "step_label": label, "auto_pick": auto, "dump_stats": stats, - "selection": { - "strategy": "z-order+last-active-popup", - "sticky_pid": self.sticky_pid, - "candidate_count": len(cands), - }, + "selection": result.selection, }, ) art.write_json(step_dir / "meta.json", meta) - art.write_json(step_dir / "tree.json", tree) + art.write_json(step_dir / "tree.json", result.tree) art.write_jsonl(step_dir / "index.jsonl", index) art.write_json(step_dir / "locatable.json", locatable) - png_ok = art.capture_window_png(info.hwnd, step_dir / "screen.png") + png_ok = False + if result.screenshot_bytes: + try: + (step_dir / "screen.png").write_bytes(result.screenshot_bytes) + png_ok = True + except Exception: + png_ok = False + elif result.use_hwnd_screenshot: + png_ok = art.capture_window_png(info.hwnd, step_dir / "screen.png") noteworthy = art.noteworthy_from_index(index) art.write_step_summary( step_dir, meta=meta, stats=stats, noteworthy=noteworthy @@ -650,11 +1773,11 @@ class InspectorApp: window=window, stats=stats, label=label, + surface=result.surface, + url=result.url, ) flow = art.load_flow(self.flow_dir) # type: ignore[arg-type] - loc_text = uia.format_locatable_text(locatable) - tree_text = uia.format_tree_text(tree) import kit_wake warm = stats.get("warmup") or {} @@ -662,53 +1785,77 @@ class InspectorApp: kit_line = kit_wake.format_kit_wake_line(kit) sparse_note = "" if stats.get("uia_sparse"): - sparse_note = "\n⚠ uia_sparse=true：可定位候选偏少，请结合 screen.png。\n" + sparse_note = "\n⚠ sparse=true：可定位候选偏少，请结合 screen.png。\n" mode = "自动" if auto else "手动" + url_line = f"url: {result.url}\n" if result.url else "" header = ( f"流程步骤: #{step_idx} `{step_name}`（{mode}）\n" - f"窗口: title={info.title!r} class={info.class_name!r} " - f"pid={info.pid} hwnd={info.hwnd}\n" + f"surface: {result.surface} ({target.surface_label(result.surface)}) " + f"driver={result.driver}\n" + f"{url_line}" + f"窗口: title={window.get('title')!r} class={window.get('class_name')!r} " + f"pid={window.get('pid')} hwnd={window.get('hwnd')}\n" + f"模式: {mode} 选窗: {sel_meta.get('strategy')}\n" f"kit: {kit_line}\n" f"软poke: findall {warm.get('findall_before')} → {warm.get('findall_after')} " f"woke={warm.get('woke')}\n" f"统计: kept={stats.get('nodes_kept')} findall={stats.get('findall_count')} " f"locatable={stats.get('locatable_count')} png={png_ok} " - f"sparse={stats.get('uia_sparse')} cold={stats.get('provider_cold')}\n" + f"sparse={stats.get('uia_sparse')} cold={stats.get('provider_cold')} " + f"merge={stats.get('merge_same_pid_allowed')}\n" f"已写入: {step_dir}\n" f"流程文档: {self.flow_dir}\n" f"{sparse_note}" f"{'=' * 60}\n" ) - body = header + "\n" + loc_text + "\n\n" + ("=" * 60) + "\n控件树:\n" + tree_text - self.root.after( - 0, - lambda i=info, f=flow, t=body, sn=step_name, a=auto: self._on_pick_ok( - i, f, t, sn, auto=a - ), + body = ( + header + + "\n" + + (result.locatable_text or "") + + "\n\n" + + ("=" * 60) + + "\n控件树:\n" + + (result.tree_text or "") + ) + self._post_ui( + self._on_pick_ok, + window, + flow, + body, + step_name, + auto=auto, + surface=result.surface, ) except Exception as exc: - err = exc - self.root.after(0, lambda e=err, a=auto: self._on_fail(e, auto=a)) + self._post_ui(self._on_fail, exc, auto=auto) threading.Thread(target=work, daemon=True).start() def _on_pick_ok( self, - info: uia.WindowInfo, + window: dict, flow: dict, text: str, step_name: str, *, auto: bool = False, + surface: str = "windows", ) -> None: self._busy = False self.flow = flow - self.last_window = info.to_dict() - self.sticky_pid = int(info.pid) + self.last_window = dict(window or {}) + if window.get("pid"): + try: + self.sticky_pid = int(window["pid"]) + except Exception: + pass self._update_flow_label() self._refresh_button_states() + url = window.get("url") or "" + url_bit = f" url={url}" if url else "" self.target_var.set( - f"本步目标: title={info.title!r} class={info.class_name!r} | step=`{step_name}`" + f"本步: surface={surface} title={window.get('title')!r} " + f"class={window.get('class_name')!r}{url_bit} | step=`{step_name}`" ) self._set_detail(text) if not auto: @@ -728,6 +1875,43 @@ class InspectorApp: # auto: either pending end, or continue loop after gap self._continue_auto_loop_after_pick() + def _on_pick_skipped( + self, + message: str, + *, + auto: bool = False, + surface: str = "browser", + url: str = "", + ) -> None: + """Blank/empty browser dump: do not write a step; keep looping if auto.""" + self._busy = False + self._refresh_button_states() + self._update_flow_label() + url_bit = f" url={url}" if url else "" + self.target_var.set(f"已跳过空页（{surface}）{url_bit}") + n = int((self.flow or {}).get("step_count") or 0) + detail = ( + f"{message}\n\n" + f"- 未写入 steps/（避免 about:blank / 空壳污染流程文档）\n" + f"- 当前流程已有步骤数: {n}\n" + f"- 打开目标站或等待页面加载后再拾取\n" + ) + if auto: + detail += f"- 自动循环将在 {_AUTO_GAP_MS // 1000} 秒后继续\n" + self._set_detail(detail) + self._set_status(message) + if not auto: + try: + self.root.lift() + self.root.attributes("-topmost", True) + except Exception: + pass + if self._pending_end: + self._pending_end = False + self._finish_end_flow() + return + self._continue_auto_loop_after_pick() + def _on_fail(self, exc: BaseException, *, auto: bool = False) -> None: self._busy = False self._refresh_button_states() @@ -742,21 +1926,54 @@ class InspectorApp: self._continue_auto_loop_after_pick() def _on_close(self) -> None: + if self._closed: + return + self._closed = True self._stop_auto_loop() self._pending_end = False - self.root.destroy() + self._stop_session() + if self.flow_dir and self.flow and self.flow.get("status") == "open": + try: + latest_flow = art.load_flow(self.flow_dir) + self.flow = art.end_flow(self.flow_dir, latest_flow) + except Exception: + pass + try: + self._persist_prefs() + except Exception: + pass + try: + kit_browser.shutdown() + except Exception: + pass + try: + self.root.destroy() + except Exception: + pass def run(self) -> None: self.captures_root.mkdir(parents=True, exist_ok=True) - self.root.mainloop() + try: + self.root.mainloop() + except KeyboardInterrupt: + pass + finally: + self._on_close() def parse_args(argv: list[str] | None = None) -> argparse.Namespace: p = argparse.ArgumentParser( - description="OpenClaw desktop UI Inspector v0.6（流程文档：开始/拾取/结束/自动串行）" + description=( + f"OpenClaw UI Inspector v{__version__}" + "（browser/windows：动作录制、检查点与 Legacy 间隔采集）" + ) ) p.add_argument("--out", type=Path, default=None, help="输出根目录") - p.add_argument("--scene", default="desktop", help="默认流程名") + p.add_argument( + "--scene", + default="", + help="默认流程名（空则按拾取模式：browser / desktop / mobile）", + ) p.add_argument("--max-depth", type=int, default=16) p.add_argument("--max-nodes", type=int, default=8000) return p.parse_args(argv) diff --git a/tools/ui_inspector/artifacts.py b/tools/ui_inspector/artifacts.py index f177df9..45ad7b5 100644 --- a/tools/ui_inspector/artifacts.py +++ b/tools/ui_inspector/artifacts.py @@ -21,14 +21,14 @@ except ImportError: _mod = importlib.util.module_from_spec(_spec) # type: ignore[arg-type] assert _spec and _spec.loader _spec.loader.exec_module(_mod) - __version__ = getattr(_mod, "__version__", "0.8.0") + __version__ = getattr(_mod, "__version__", "0.16.0") except Exception: - __version__ = "0.8.0" + __version__ = "0.16.0" def _local_ts() -> str: - now = datetime.now().astimezone() - return now.strftime("%Y%m%dT%H%M%S%z") + """Local wall-clock stamp for capture folder names: YYYYMMDDHHMMSS.""" + return datetime.now().astimezone().strftime("%Y%m%d%H%M%S") def safe_slug(text: str, *, fallback: str = "session") -> str: @@ -405,11 +405,14 @@ def build_meta( scene: str, skill_root: Optional[str] = None, extra: Optional[dict[str, Any]] = None, + surface: str = "windows", + driver: str = "uiautomation", + url: str = "", ) -> dict[str, Any]: now = datetime.now().astimezone() meta = { "schema": "openclaw.ui_inspector.capture.v1", - "surface": "windows", + "surface": surface or "windows", "captured_at": now.isoformat(), "scene": scene, "skill_root": skill_root, @@ -419,6 +422,7 @@ def build_meta( "process_name": window.get("process_name"), "pid": window.get("pid"), "hwnd": window.get("hwnd"), + "url": url or window.get("url") or "", }, "environment": { "os": "windows", @@ -427,7 +431,7 @@ def build_meta( "tool": { "name": "openclaw-ui-inspector", "version": __version__, - "driver": "uiautomation", + "driver": driver or "uiautomation", }, } if extra: @@ -486,14 +490,18 @@ def capture_window_png(hwnd: int, dest: Path) -> bool: def create_flow( captures_root: Path, *, - flow_name: str = "desktop-flow", + flow_name: str = "desktop", + recording_mode: str = "auto", ) -> Path: - """开始流程：新建一份流程文档目录。""" + """开始流程：新建一份流程文档目录（v1 steps + 可选 v2 资产目录）。""" captures_root.mkdir(parents=True, exist_ok=True) slug = safe_slug(flow_name, fallback="flow") flow_dir = captures_root / f"{_local_ts()}_{slug}" flow_dir.mkdir(parents=True, exist_ok=False) (flow_dir / "steps").mkdir(parents=True, exist_ok=True) + # v2 dual-write directories (empty until recorder writes) + for sub in ("states", "observations", "actions", "elements", "journal", "diagnostics"): + (flow_dir / sub).mkdir(parents=True, exist_ok=True) now = datetime.now().astimezone().isoformat() flow = { "schema": "openclaw.ui_inspector.flow.v1", @@ -508,6 +516,21 @@ def create_flow( "name": "openclaw-ui-inspector", "version": __version__, }, + # Optional v2 pointers — old readers ignore unknown fields. + "v2": { + "recording_mode": recording_mode or "auto", + "action_flow": "action_flow.json", + "element_catalog": "element_catalog.json", + "states_dir": "states/", + "observations_dir": "observations/", + "actions_dir": "actions/", + "elements_dir": "elements/", + "raw_event_journal": "journal/raw_events.jsonl", + "pages": "pages.json", + "diagnostics_dir": "diagnostics/", + "validation": "diagnostics/validation.json", + "replay_report": "diagnostics/replay_report.json", + }, } write_json(flow_dir / "flow.json", flow) write_flow_summary(flow_dir, flow) @@ -565,12 +588,16 @@ def register_step( window: dict[str, Any], stats: dict[str, Any], label: Optional[str] = None, + surface: str = "windows", + url: str = "", ) -> dict[str, Any]: entry = { "index": step_index, "name": step_name, "label": label or step_name, "captured_at": datetime.now().astimezone().isoformat(), + "surface": surface or "windows", + "url": url or window.get("url") or "", "window_title": window.get("title"), "window_class": window.get("class_name"), "pid": window.get("pid"), @@ -621,6 +648,9 @@ def write_flow_summary(flow_dir: Path, flow: dict[str, Any]) -> None: lines.append("") if s.get("label"): lines.append(f"- label: `{s.get('label')}`") + lines.append(f"- surface: `{s.get('surface') or 'windows'}`") + if s.get("url"): + lines.append(f"- url: `{s.get('url')}`") lines.append( f"- window: `{s.get('window_title')}` / `{s.get('window_class')}`" ) @@ -636,7 +666,10 @@ def write_flow_summary(flow_dir: Path, flow: dict[str, Any]) -> None: "", "- 一个流程对应本目录一份文档（`flow.json` + `SUMMARY.md` + `steps/`）。", "- 每拾取一页，在 `steps/` 下新增一步；给 AI 时优先读本 `SUMMARY.md` 与各步 `locatable.json`。", - "- Name ≠ 界面中文时请看 Value。", + "- `surface=windows` 为桌面 UIA；`surface=browser` 为 Playwright 整页 dump。", + "- Name ≠ 界面中文时请看 Value；浏览器步优先看 `primary_locator` / `playwright_candidates`。", + "- v2（若存在）：`action_flow.json` + `element_catalog.json` + `states/` + `actions/` + `elements/`。", + "- 给 AI 编程工具时优先提供 `action_flow.json` 与 `element_catalog.json`，诊断再用 steps/ 原始树。", "", ] ) @@ -651,13 +684,18 @@ def write_step_summary( noteworthy: Optional[list[dict[str, Any]]] = None, ) -> None: app = meta.get("app") or {} + tool = meta.get("tool") or {} lines = [ "# Step Summary", "", f"- captured_at: `{meta.get('captured_at')}`", + f"- surface: `{meta.get('surface')}` driver: `{tool.get('driver')}`", f"- window: `{app.get('title')}` / `{app.get('class_name')}` pid=`{app.get('pid')}`", "", ] + if app.get("url"): + lines.append(f"- url: `{app.get('url')}`") + lines.append("") if stats: lines.extend( [ @@ -673,9 +711,11 @@ def write_step_summary( if noteworthy: lines.extend(["## Notable", ""]) for item in noteworthy[:40]: + primary = (item.get("primary_locator") or {}).get("expr") or "" + extra = f" locator=`{primary}`" if primary else "" lines.append( f"- `{item.get('control_type')}` Name=`{item.get('name')}` " - f"Value=`{item.get('value')}`" + f"Value=`{item.get('value') or item.get('text')}`{extra}" ) lines.append("") lines.extend( @@ -689,6 +729,7 @@ def write_step_summary( (step_dir / "SUMMARY.md").write_text("\n".join(lines), encoding="utf-8") + # Back-compat aliases used by older smoke scripts def create_session_dir(captures_root: Path, *, scene: str = "desktop") -> Path: return create_flow(captures_root, flow_name=scene) diff --git a/tools/ui_inspector/compatibility/__init__.py b/tools/ui_inspector/compatibility/__init__.py new file mode 100644 index 0000000..49ec57b --- /dev/null +++ b/tools/ui_inspector/compatibility/__init__.py @@ -0,0 +1,2 @@ +"""Backward-compatible readers and entry points.""" + diff --git a/tools/ui_inspector/compatibility/v1_reader.py b/tools/ui_inspector/compatibility/v1_reader.py new file mode 100644 index 0000000..fbd821b --- /dev/null +++ b/tools/ui_inspector/compatibility/v1_reader.py @@ -0,0 +1,6 @@ +"""Legacy flow.json reader.""" + +from artifacts import load_flow + +__all__ = ["load_flow"] + diff --git a/tools/ui_inspector/compatibility/v1_writer.py b/tools/ui_inspector/compatibility/v1_writer.py new file mode 100644 index 0000000..f46ca1e --- /dev/null +++ b/tools/ui_inspector/compatibility/v1_writer.py @@ -0,0 +1,6 @@ +"""Legacy steps/ writer exports.""" + +from artifacts import add_step_dir, create_flow, save_flow + +__all__ = ["add_step_dir", "create_flow", "save_flow"] + diff --git a/tools/ui_inspector/core/__init__.py b/tools/ui_inspector/core/__init__.py new file mode 100644 index 0000000..ad8b7f7 --- /dev/null +++ b/tools/ui_inspector/core/__init__.py @@ -0,0 +1,2 @@ +"""Surface-independent UI Inspector domain and recording pipeline.""" + diff --git a/tools/ui_inspector/core/models/__init__.py b/tools/ui_inspector/core/models/__init__.py new file mode 100644 index 0000000..d957dc7 --- /dev/null +++ b/tools/ui_inspector/core/models/__init__.py @@ -0,0 +1,16 @@ +"""Canonical model exports. + +The compatibility ``models`` package remains importable while callers migrate +to this namespace. +""" + +from models.action import ActionFlow, ActionRecord +from models.element import ElementCatalog, ElementRef +from models.locator import LocatorCandidate +from models.observation import UIObservation +from models.state import UIState + +__all__ = [ + "ActionFlow", "ActionRecord", "ElementCatalog", "ElementRef", + "LocatorCandidate", "UIObservation", "UIState", +] diff --git a/tools/ui_inspector/core/pipeline/__init__.py b/tools/ui_inspector/core/pipeline/__init__.py new file mode 100644 index 0000000..46f4c6e --- /dev/null +++ b/tools/ui_inspector/core/pipeline/__init__.py @@ -0,0 +1,6 @@ +"""Shared event, action, and evidence pipeline.""" + +from .event_journal import EventJournal + +__all__ = ["EventJournal"] + diff --git a/tools/ui_inspector/core/pipeline/event_journal.py b/tools/ui_inspector/core/pipeline/event_journal.py new file mode 100644 index 0000000..c6622dd --- /dev/null +++ b/tools/ui_inspector/core/pipeline/event_journal.py @@ -0,0 +1,49 @@ +"""Append-only raw interaction journal. + +Raw events are facts. They are persisted before semantic reduction so a +future reducer can rebuild Action Flow without asking the user to repeat the +workflow. +""" + +from __future__ import annotations + +import json +import threading +from dataclasses import asdict, is_dataclass +from datetime import datetime +from pathlib import Path +from typing import Any + +from redaction import sanitize_artifact + + +class EventJournal: + def __init__(self, flow_dir: Path) -> None: + self.path = Path(flow_dir) / "journal" / "raw_events.jsonl" + self.path.parent.mkdir(parents=True, exist_ok=True) + self._lock = threading.Lock() + self._sequence = 0 + + def append(self, event: Any, *, channel: str = "surface") -> int: + if is_dataclass(event) and not isinstance(event, type): + payload = asdict(event) + elif isinstance(event, dict): + payload = dict(event) + else: + payload = dict(getattr(event, "__dict__", {}) or {}) + with self._lock: + self._sequence += 1 + record = { + "journal_sequence": self._sequence, + "received_at": datetime.now().astimezone().isoformat(), + "channel": channel, + "event": sanitize_artifact(payload), + } + line = json.dumps( + record, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ) + with self.path.open("a", encoding="utf-8", newline="\n") as stream: + stream.write(line + "\n") + stream.flush() + return self._sequence + diff --git a/tools/ui_inspector/core/policies/__init__.py b/tools/ui_inspector/core/policies/__init__.py new file mode 100644 index 0000000..8bf68e9 --- /dev/null +++ b/tools/ui_inspector/core/policies/__init__.py @@ -0,0 +1,7 @@ +"""Surface-independent decision policies.""" + +from .element_identity import element_identity_key +from .target_scoring import resolve_browser_target + +__all__ = ["element_identity_key", "resolve_browser_target"] + diff --git a/tools/ui_inspector/core/policies/element_identity.py b/tools/ui_inspector/core/policies/element_identity.py new file mode 100644 index 0000000..9adcdc3 --- /dev/null +++ b/tools/ui_inspector/core/policies/element_identity.py @@ -0,0 +1,62 @@ +"""Stable element identity independent of transient value/focus classes.""" + +from __future__ import annotations + +import hashlib +import json +import re +from typing import Any +from urllib.parse import urlsplit + + +def _origin(url: str) -> str: + try: + parts = urlsplit(url or "") + return f"{parts.scheme}://{parts.netloc}" if parts.netloc else "" + except Exception: + return "" + + +def _structural_css(css: str) -> str: + # Classes frequently encode focus/hover/build hashes. Keep tag depth and + # stable IDs; that distinguishes nested anonymous nodes without freezing a + # generated class list into identity. + parts = [] + for raw in (css or "").split(">"): + part = raw.strip() + tag_match = re.match(r"^[A-Za-z][\w-]*", part) + id_match = re.search(r"#[A-Za-z0-9_-]+", part) + tag = tag_match.group(0).lower() if tag_match else "*" + parts.append(tag + (id_match.group(0) if id_match else "")) + return ">".join(parts) + + +def element_identity_key( + element: dict[str, Any], *, surface: str, page_url: str = "", frame_name: str = "" +) -> str: + el = dict(element or {}) + role = str(el.get("control_type") or el.get("role") or el.get("tag") or "") + anchor = "" + for key in ( + "test_id", "automation_id", "id", "name_attr", "href", + "placeholder", "aria_label", "label", + ): + value = str(el.get(key) or "").strip() + if value: + anchor = f"{key}:{value}" + break + if not anchor and not el.get("contenteditable"): + name = str(el.get("semantic_name") or el.get("name") or "").strip() + if name and len(name) <= 96: + anchor = f"name:{name}" + if not anchor: + anchor = f"structure:{_structural_css(str(el.get('css') or ''))}" + payload = { + "surface": surface or "", + "origin": _origin(page_url) if surface == "browser" else "", + "frame": frame_name or "", + "role": role.lower(), + "anchor": anchor, + } + raw = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(raw.encode("utf-8")).hexdigest() diff --git a/tools/ui_inspector/core/policies/target_scoring.py b/tools/ui_inspector/core/policies/target_scoring.py new file mode 100644 index 0000000..8a0a68f --- /dev/null +++ b/tools/ui_inspector/core/policies/target_scoring.py @@ -0,0 +1,145 @@ +"""Distance- and geometry-aware target selection. + +The resolver never relies on a product selector. It ranks the composed path +using native semantics, explicit accessibility metadata, distance from the hit +node, and geometry. Huge ancestors are evidence containers, not controls. +""" + +from __future__ import annotations + +import re +from typing import Any, Iterable + +_NATIVE_TAGS = {"button", "a", "input", "textarea", "select", "summary", "label"} +_INTERACTIVE_ROLES = { + "button", "link", "textbox", "combobox", "checkbox", "radio", + "menuitem", "option", "tab", "switch", "slider", "spinbutton", +} +_ACTION_CLASS_TOKENS = { + "button", "btn", "submit", "send", "publish", "post", "play", + "pause", "action", "click", "toggle", "menu", "option", "tab", +} +_GRAPHIC_LEAF_TAGS = {"path", "use", "g", "svg"} + + +def _tokens(value: str) -> set[str]: + value = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", value or "") + return {x.lower() for x in re.split(r"[^A-Za-z0-9]+", value) if x} + + +def _area(rect: dict[str, Any]) -> float: + try: + return max(0.0, float(rect.get("width") or 0)) * max( + 0.0, float(rect.get("height") or 0) + ) + except (TypeError, ValueError): + return 0.0 + + +def _contains( + rect: dict[str, Any], x: int | None, y: int | None, *, slop: float = 12.0 +) -> bool: + if x is None or y is None or not rect: + return True + try: + return ( + float(rect.get("left")) - slop <= x <= float(rect.get("right")) + slop + and float(rect.get("top")) - slop <= y <= float(rect.get("bottom")) + slop + ) + except (TypeError, ValueError): + return True + + +def resolve_browser_target( + hit_target: dict[str, Any], + candidates: Iterable[dict[str, Any]], + *, + x: int | None = None, + y: int | None = None, +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """Return the best actionable node and an auditable scored candidate list.""" + chain = [dict(item or {}) for item in candidates if isinstance(item, dict)] + if not chain: + chain = [dict(hit_target or {})] + hit = dict(hit_target or chain[0] or {}) + hit_area = max(1.0, _area(hit.get("rect") or {})) + scored: list[dict[str, Any]] = [] + + for depth, item in enumerate(chain): + tag = str(item.get("tag") or "").lower() + role = str(item.get("role") or "").lower() + explicit_role = str(item.get("explicit_role") or "").lower() + rect = dict(item.get("rect") or {}) + area = _area(rect) + score = 24.0 if depth == 0 else 0.0 + reasons: list[str] = [f"depth={depth}"] + + if tag in _NATIVE_TAGS: + score = max(score, 100.0) + reasons.append("native_control") + if item.get("contenteditable_attr"): + score = max(score, 99.0) + reasons.append("editable_root") + elif item.get("contenteditable"): + score = max(score, 82.0) + reasons.append("editable_descendant") + if explicit_role in _INTERACTIVE_ROLES: + score = max(score, 97.0) + reasons.append("explicit_interactive_role") + elif role in _INTERACTIVE_ROLES: + score = max(score, 88.0) + reasons.append("computed_interactive_role") + if item.get("test_id") or item.get("aria_label"): + score = max(score, 91.0) + reasons.append("stable_semantic_attribute") + if item.get("tabindex") not in (None, ""): + score = max(score, 76.0) + reasons.append("keyboard_focusable") + if _tokens(str(item.get("class_name") or "")) & _ACTION_CLASS_TOKENS: + score = max(score, 82.0) + reasons.append("action_class_token") + if str(item.get("cursor") or "").lower() == "pointer": + score = max(score, 68.0) + reasons.append("pointer_cursor") + + # Target selection and locator quality are one decision. Event-time + # counts are captured before a click can navigate or mutate the DOM. + counts = [ + item.get("stable_css_count"), + item.get("href_count"), + item.get("css_count"), + ] + known_counts = [int(value) for value in counts if value is not None] + if 1 in known_counts: + score += 22.0 + reasons.append("event_time_unique_locator") + elif known_counts and min(known_counts) > 1: + score -= 18.0 + reasons.append("event_time_ambiguous_locator") + + if tag in _GRAPHIC_LEAF_TAGS: + score -= 28.0 + reasons.append("graphic_leaf") + elif ( + str(item.get("cursor") or "").lower() == "pointer" + and 20.0 <= float(rect.get("width") or 0.0) <= 96.0 + and 20.0 <= float(rect.get("height") or 0.0) <= 96.0 + ): + score += 14.0 + reasons.append("compact_clickable_wrapper") + + score -= depth * 3.5 + if not _contains(rect, x, y): + score -= 100.0 + reasons.append("outside_pointer") + if area and area > max(hit_area * 25.0, 200_000.0): + score -= 70.0 + reasons.append("oversized_ancestor") + elif area and area > max(hit_area * 8.0, 60_000.0): + score -= 25.0 + reasons.append("large_ancestor") + + scored.append({"depth": depth, "score": round(score, 3), "reasons": reasons, "target": item}) + + winner = max(scored, key=lambda row: (row["score"], -row["depth"])) + return dict(winner["target"]), scored diff --git a/tools/ui_inspector/desktop_uia.py b/tools/ui_inspector/desktop_uia.py index d5d7377..ba5f743 100644 --- a/tools/ui_inspector/desktop_uia.py +++ b/tools/ui_inspector/desktop_uia.py @@ -508,6 +508,70 @@ def prefer_last_active_popup(info: WindowInfo) -> WindowInfo: return info +def window_info_under_cursor( + *, + exclude_hwnds: Optional[set[int]] = None, +) -> Optional[WindowInfo]: + """光标下的顶层窗（不 Activate、不点击）。光标在 Inspector 上则返回 None。""" + exclude = {int(h) for h in (exclude_hwnds or set()) if h} + exclude |= {get_toplevel_hwnd(h) for h in list(exclude)} + pt = wintypes.POINT() + if not user32.GetCursorPos(ctypes.byref(pt)): + return None + hwnd = int(user32.WindowFromPoint(pt) or 0) + if not hwnd: + return None + top = get_toplevel_hwnd(hwnd) + if not top or top in exclude: + return None + if not user32.IsWindowVisible(top) or user32.IsIconic(top) or _is_cloaked(top): + return None + try: + info = window_info_from_hwnd(top) + except Exception: + return None + if is_inspector_window(info): + return None + if "UI Inspector" in (info.title or "") or "OpenClaw UI Inspector" in (info.title or ""): + return None + return info + + +def find_wecom_search_result_window( + *, + pid: Optional[int] = None, + exclude_hwnds: Optional[set[int]] = None, +) -> Optional[WindowInfo]: + """同进程可见的 SearchResultWindow2（搜索联系人浮层），不要求前台焦点。""" + exclude = {int(h) for h in (exclude_hwnds or set()) if h} + exclude |= {get_toplevel_hwnd(h) for h in list(exclude)} + best: Optional[WindowInfo] = None + best_area = -1 + for hwnd in iter_toplevel_hwnds_zorder(): + if hwnd in exclude: + continue + if not user32.IsWindowVisible(hwnd) or user32.IsIconic(hwnd) or _is_cloaked(hwnd): + continue + cls_buf = ctypes.create_unicode_buffer(256) + user32.GetClassNameW(hwnd, cls_buf, 256) + if (cls_buf.value or "") != "SearchResultWindow2": + continue + try: + info = window_info_from_hwnd(hwnd) + except Exception: + continue + if pid is not None and int(info.pid) != int(pid): + continue + rect = _window_rect(hwnd) or {} + area = int(rect.get("width") or 0) * int(rect.get("height") or 0) + if area < 40 * 40: + continue + if area > best_area: + best_area = area + best = info + return best + + def foreground_toplevel_hwnd(*, exclude_hwnds: Optional[set[int]] = None) -> Optional[int]: """True foreground top-level hwnd (GetForegroundWindow), if capturable.""" exclude = {int(h) for h in (exclude_hwnds or set()) if h} @@ -530,16 +594,38 @@ def find_capture_target( *, exclude_hwnds: Optional[set[int]] = None, prefer_pid: Optional[int] = None, + under_cursor: bool = False, + prefer_wecom_search_popup: bool = True, ) -> tuple[WindowInfo, list[WindowInfo]]: """ - Pick the frontmost capturable window. + Pick a capturable top-level window. - Priority: - 1) GetForegroundWindow (true frontmost), if capturable + Default (under_cursor=False) keeps legacy behaviour: + 1) GetForegroundWindow, if capturable 2) else Z-order top capturable candidate - Optional prefer_pid only filters when sticky (same front app pid); - never jumps to another app just because it is larger. + 3) prefer_last_active_popup (same pid) + + Optional: + - under_cursor: 光标下顶层窗优先（不点击、不抢焦点）；失败则回退前台逻辑 + - prefer_wecom_search_popup: 若候选为企微主窗且存在 SearchResultWindow2，改采浮层 """ + info, cands, _meta = resolve_capture_target( + exclude_hwnds=exclude_hwnds, + prefer_pid=prefer_pid, + under_cursor=under_cursor, + prefer_wecom_search_popup=prefer_wecom_search_popup, + ) + return info, cands + + +def resolve_capture_target( + *, + exclude_hwnds: Optional[set[int]] = None, + prefer_pid: Optional[int] = None, + under_cursor: bool = False, + prefer_wecom_search_popup: bool = True, +) -> tuple[WindowInfo, list[WindowInfo], dict[str, Any]]: + """与 find_capture_target 相同，额外返回 selection meta（strategy 链）。""" candidates = list_capturable_candidates(exclude_hwnds=exclude_hwnds) if not candidates: raise RuntimeError( @@ -548,21 +634,31 @@ def find_capture_target( by_hwnd = {int(c.hwnd): c for c in candidates} chosen: Optional[WindowInfo] = None + strategy_parts: list[str] = [] - fg = foreground_toplevel_hwnd(exclude_hwnds=exclude_hwnds) - if fg and fg in by_hwnd: - # Frontmost always wins — never replace with a larger/sticky other app. - chosen = by_hwnd[fg] - elif fg: - # Foreground exists but filtered from candidates — still try to describe it. - try: - info = window_info_from_hwnd(fg) - if not is_inspector_window(info) and not ( - is_system_chrome_window(info) and not (info.title or "").strip() - ): - chosen = info - except Exception: - chosen = None + if under_cursor: + cur = window_info_under_cursor(exclude_hwnds=exclude_hwnds) + if cur is not None: + chosen = cur + strategy_parts.append("under_cursor") + + if chosen is None: + fg = foreground_toplevel_hwnd(exclude_hwnds=exclude_hwnds) + if fg and fg in by_hwnd: + # Frontmost always wins — never replace with a larger/sticky other app. + chosen = by_hwnd[fg] + strategy_parts.append("foreground") + elif fg: + # Foreground exists but filtered from candidates — still try to describe it. + try: + info = window_info_from_hwnd(fg) + if not is_inspector_window(info) and not ( + is_system_chrome_window(info) and not (info.title or "").strip() + ): + chosen = info + strategy_parts.append("foreground_raw") + except Exception: + chosen = None if chosen is None: # No usable foreground: Z-order front. prefer_pid only as soft filter here. @@ -570,15 +666,35 @@ def find_capture_target( for c in candidates: if int(c.pid) == int(prefer_pid): chosen = c + strategy_parts.append("sticky_pid") break if chosen is None: chosen = candidates[0] + strategy_parts.append("zorder") # Owned popup of the *same* front window only (e.g. search panel), never another app popup = prefer_last_active_popup(chosen) - if int(popup.pid) == int(chosen.pid): + if int(popup.pid) == int(chosen.pid) and int(popup.hwnd) != int(chosen.hwnd): chosen = popup - return chosen, candidates + strategy_parts.append("last_active_popup") + + # 企微：主窗为前台时，若搜索浮层仍打开，优先采浮层（无需点击进会话） + if prefer_wecom_search_popup and (chosen.class_name or "") == "WeWorkWindow": + search_win = find_wecom_search_result_window( + pid=int(chosen.pid), exclude_hwnds=exclude_hwnds + ) + if search_win is not None: + chosen = search_win + strategy_parts.append("wecom_search_popup") + + meta = { + "strategy": "+".join(strategy_parts) if strategy_parts else "unknown", + "under_cursor": bool(under_cursor), + "prefer_wecom_search_popup": bool(prefer_wecom_search_popup), + "chosen_class": chosen.class_name, + "chosen_hwnd": int(chosen.hwnd), + } + return chosen, candidates, meta def wait_frontmost_target( @@ -632,8 +748,6 @@ def prepare_frontmost_for_capture( 企微：先静默调用 kit ``ensure_ui_ready("wecom")``（与技能同路径），再轻量 poke。 """ - import kit_wake - info, cands, wait_meta = wait_frontmost_target( exclude_hwnds=exclude_hwnds, timeout_sec=timeout_sec ) @@ -642,28 +756,29 @@ def prepare_frontmost_for_capture( # Keep operator size/position; only restore if minimized. act = activate_main_hwnd(info.hwnd, maximize=False) + # Product adapters may need to make their accessibility provider ready + # before the operator starts typing. Running this on the app's existing + # startup worker keeps Tk responsive and prevents the first input + # transaction from racing a 10-20 second background warmup. + import kit_wake + kit_report = kit_wake.auto_ensure_if_wecom( class_name=info.class_name or "", title=info.title or "", hwnd=info.hwnd, - force=True, ) + findall = kit_report.get("findall") if kit_report.get("ok") else None + provider_cold = bool(kit_report.get("applicable") and not kit_report.get("ok")) - warm = warmup_uia_provider( - info.hwnd, interact_window=False, allow_foreground_nudge=False - ) - n = int(warm.get("findall_after") or findall_count_for_hwnd(info.hwnd) or 0) - if kit_report.get("ok") and int(kit_report.get("findall") or 0) > n: - n = int(kit_report["findall"]) return { "ready": True, # frontmost resolved => ready to capture that window "window": info.to_dict(), "candidates": len(cands), - "findall": n, - "uia_sparse": n < 8, - "provider_cold": is_provider_cold(info.hwnd, findall=n), + "findall": findall, + "uia_sparse": bool(provider_cold), + "provider_cold": provider_cold, "activate": act, - "warmup": warm, + "warmup": {"mode": "startup_adapter" if kit_report.get("applicable") else "deferred"}, "kit_wake": kit_report, "wait": wait_meta, "strategy": "foreground_only", @@ -933,6 +1048,7 @@ def dump_window_tree( warmup_mode: - full: MSAA/UIA poke on target; restore only if minimized - light: ElementFromHandle poke only (no ShowWindow) + - probe: light poke + FindAll, but never invoke a blocking product wake - off: no provider warmup Child / same-PID merges never call ShowWindow (light poke at most). """ @@ -942,7 +1058,7 @@ def dump_window_tree( mode = (warmup_mode or "full").lower() if not warmup: mode = "off" - if mode not in {"full", "light", "off"}: + if mode not in {"full", "light", "probe", "off"}: mode = "full" kit_report: dict[str, Any] = {"applicable": False, "skipped": True, "reason": "not_checked"} @@ -950,7 +1066,11 @@ def dump_window_tree( winfo_for_kit = window_info_from_hwnd(hwnd) except Exception: winfo_for_kit = None - if winfo_for_kit is not None: + # 搜索浮层稀疏时勿合并主窗会话列表，否则验收会误判「已有数据」 + allow_same_pid_merge = bool(merge_same_pid_if_sparse) + if winfo_for_kit is not None and (winfo_for_kit.class_name or "") == "SearchResultWindow2": + allow_same_pid_merge = False + if winfo_for_kit is not None and mode not in {"off", "probe"}: hint = findall_count_for_hwnd(hwnd) kit_report = kit_wake.auto_ensure_if_wecom( class_name=winfo_for_kit.class_name or "", @@ -977,7 +1097,7 @@ def dump_window_tree( warmup_report = warmup_uia_provider( hwnd, interact_window=True, allow_foreground_nudge=True ) - elif mode == "light": + elif mode in {"light", "probe"}: warmup_report = poke_uia_provider(hwnd) def _run_dump() -> tuple[dict[str, Any], list[dict[str, Any]], dict[str, Any]]: @@ -1035,7 +1155,7 @@ def dump_window_tree( t = dict(t) t["children"] = children_nodes - if use_findall: + if use_findall and mode != "off": found = collect_findall_flat(root_ctrl, max_nodes=max_nodes) st["findall_count"] = len(found) existing = {r.get("path_hint") for r in idx} @@ -1047,6 +1167,13 @@ def dump_window_tree( return t, idx, st tree, index, stats = _run_dump() + if winfo_for_kit is not None: + kit_wake.note_provider_warm( + class_name=winfo_for_kit.class_name or "", + title=winfo_for_kit.title or "", + hwnd=hwnd, + findall=int(stats.get("findall_count") or 0), + ) stats["warmup"] = warmup_report stats["warmup_mode"] = mode stats["kit_wake"] = kit_report @@ -1061,7 +1188,7 @@ def dump_window_tree( ) # 仍冷且目标为企微：强制再 ensure 一次（绕过短时 cache）后重 dump - if stats["provider_cold"] and winfo_for_kit is not None and kit_wake.looks_like_wecom( + if mode not in {"off", "probe"} and stats["provider_cold"] and winfo_for_kit is not None and kit_wake.looks_like_wecom( class_name=winfo_for_kit.class_name or "", title=winfo_for_kit.title or "", ): @@ -1117,7 +1244,9 @@ def dump_window_tree( # Sparse fallback: light poke + FindAll on a few large same-pid top-levels. # Includes WeWorkWindow when current target is a thin popup. - if merge_same_pid_if_sparse and stats["uia_sparse"]: + # Never merge into SearchResultWindow2 (see allow_same_pid_merge). + stats["merge_same_pid_allowed"] = allow_same_pid_merge + if allow_same_pid_merge and stats["uia_sparse"]: try: info = window_info_from_hwnd(hwnd) merged_n = 0 @@ -1329,15 +1458,20 @@ def format_tree_text(tree: dict[str, Any], *, max_lines: int = 2000) -> str: return "\n".join(lines) -# Back-compat alias used by older app versions -def resolve_capture_target( +# Back-compat: older callers expected a single WindowInfo (foreground only). +# Do NOT name this resolve_capture_target — that name is the full 3-tuple API above. +def resolve_frontmost_window( *, self_hwnd: int = 0, poll_sec: float = 0.15, wait_sec: float = 2.0, ) -> WindowInfo: _ = poll_sec, wait_sec - info, _cands = find_capture_target(exclude_hwnds={int(self_hwnd)} if self_hwnd else None) + info, _cands = find_capture_target( + exclude_hwnds={int(self_hwnd)} if self_hwnd else None, + under_cursor=False, + prefer_wecom_search_popup=False, + ) return info diff --git a/tools/ui_inspector/diagnostics/__init__.py b/tools/ui_inspector/diagnostics/__init__.py new file mode 100644 index 0000000..d70cdb9 --- /dev/null +++ b/tools/ui_inspector/diagnostics/__init__.py @@ -0,0 +1,6 @@ +"""Capture validation and timeline diagnostics.""" + +from .capture_validator import validate_capture +from .replay_preflight import build_replay_preflight, write_replay_preflight + +__all__ = ["validate_capture", "build_replay_preflight", "write_replay_preflight"] diff --git a/tools/ui_inspector/diagnostics/capture_validator.py b/tools/ui_inspector/diagnostics/capture_validator.py new file mode 100644 index 0000000..d210572 --- /dev/null +++ b/tools/ui_inspector/diagnostics/capture_validator.py @@ -0,0 +1,134 @@ +"""Product-neutral capture quality checks.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +def _read(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def validate_capture(flow_dir: Path) -> dict[str, Any]: + root = Path(flow_dir) + issues: list[dict[str, Any]] = [] + actions = sorted( + (_read(path) for path in (root / "actions").glob("*.json")), + key=lambda item: int(item.get("sequence") or 0), + ) + expected = list(range(1, len(actions) + 1)) + actual = [int(item.get("sequence") or 0) for item in actions] + if actual != expected: + issues.append({"severity": "error", "code": "ACTION_SEQUENCE_GAP", "actual": actual}) + elements = { + item.get("id"): item + for item in (_read(path) for path in (root / "elements").glob("*.json")) + if item.get("id") + } + interactive_types = {"click", "fill", "type", "press_key", "select", "toggle"} + previous_action: dict[str, Any] | None = None + for action in actions: + target = dict(action.get("target_snapshot") or {}) + rect = dict(target.get("rect") or target.get("bounds") or {}) + area = float(rect.get("width") or 0) * float(rect.get("height") or 0) + if action.get("action_type") in {"click", "fill", "select", "toggle"} and not target: + issues.append({"severity": "error", "code": "ACTION_TARGET_MISSING", "action_id": action.get("id")}) + if action.get("action_type") == "click" and area > 1_000_000: + issues.append({"severity": "error", "code": "CLICK_TARGET_OVERSIZED", "action_id": action.get("id"), "area": area}) + target_name = str(target.get("name") or target.get("automation_id") or "").lower() + if ( + action.get("surface") == "windows" + and action.get("action_type") == "click" + and str(target.get("control_type") or "") in {"TreeControl", "PaneControl", "GroupControl"} + and any(token in target_name for token in ("search_result", "listview", "list_view", "watermark", "container", "layout")) + ): + issues.append({"severity": "warning", "code": "CLICK_TARGET_GENERIC_CONTAINER", "action_id": action.get("id"), "target_name": target_name}) + if ( + action.get("surface") == "windows" + and action.get("action_type") == "press_key" + and (str(target.get("control_type") or "") == "WindowControl" or area > 1_000_000) + ): + issues.append({"severity": "warning", "code": "KEY_TARGET_ROOT_WINDOW", "action_id": action.get("id")}) + if action.get("action_type") == "click" and str(target.get("tag") or "").lower() in {"path", "use", "g", "svg"}: + issues.append({"severity": "warning", "code": "GRAPHIC_LEAF_TARGET", "action_id": action.get("id"), "tag": target.get("tag")}) + evidence = dict(action.get("evidence") or {}) + if not action.get("after_observation_id") and not evidence.get("eventual_observation_id"): + issues.append({"severity": "warning", "code": "ACTION_EVIDENCE_MISSING", "action_id": action.get("id")}) + if evidence.get("locator_unique") is False: + issues.append({"severity": "warning", "code": "LOCATOR_NOT_UNIQUE", "action_id": action.get("id")}) + if action.get("surface") == "browser" and action.get("action_type") in interactive_types: + element_id = action.get("target_element_id") + if target and not element_id: + issues.append({"severity": "warning", "code": "TARGET_ELEMENT_REFERENCE_MISSING", "action_id": action.get("id")}) + element = elements.get(element_id) if element_id else None + primary = dict((element or {}).get("primary_locator") or {}) + if element_id and (not primary or not primary.get("production_eligible")): + issues.append({"severity": "warning", "code": "ACTION_PRODUCTION_LOCATOR_MISSING", "action_id": action.get("id"), "element_id": element_id}) + elif primary and float(primary.get("stability_score") or 0.0) < 0.55: + issues.append({"severity": "warning", "code": "LOW_STABILITY_LOCATOR", "action_id": action.get("id"), "element_id": element_id, "strategy": primary.get("strategy"), "stability_score": primary.get("stability_score")}) + raw_meta = dict((action.get("raw_event") or {}).get("meta") or {}) + if not raw_meta.get("event_id"): + issues.append({"severity": "warning", "code": "ACTION_RAW_EVENT_LINK_MISSING", "action_id": action.get("id")}) + if action.get("surface") == "windows" and action.get("action_type") in interactive_types: + element_id = action.get("target_element_id") + if target and not element_id: + issues.append({"severity": "warning", "code": "TARGET_ELEMENT_REFERENCE_MISSING", "action_id": action.get("id")}) + element = elements.get(element_id) if element_id else None + primary = dict((element or {}).get("primary_locator") or {}) + if element_id and not primary: + issues.append({"severity": "warning", "code": "ACTION_PRODUCTION_LOCATOR_MISSING", "action_id": action.get("id"), "element_id": element_id}) + if previous_action and action.get("action_type") == "fill" and previous_action.get("action_type") == "fill": + same_target = bool(action.get("target_element_id")) and action.get("target_element_id") == previous_action.get("target_element_id") + old_text = str((previous_action.get("input") or {}).get("text") or "") + new_text = str((action.get("input") or {}).get("text") or "") + old_ms = ((previous_action.get("raw_event") or {}).get("meta") or {}).get("source_ts_ms") + new_ms = ((action.get("raw_event") or {}).get("meta") or {}).get("source_ts_ms") + close = False + try: + close = 0 <= float(new_ms) - float(old_ms) <= 2000 + except (TypeError, ValueError): + pass + if same_target and old_text and new_text.startswith(old_text) and new_text != old_text and close: + issues.append({"severity": "warning", "code": "DUPLICATE_FILL_TRANSACTION", "action_id": action.get("id"), "previous_action_id": previous_action.get("id"), "element_id": action.get("target_element_id")}) + previous_action = action + journal = root / "journal" / "raw_events.jsonl" + if not journal.exists(): + issues.append({"severity": "error", "code": "RAW_EVENT_JOURNAL_MISSING"}) + if not actions: + issues.append({"severity": "error", "code": "ZERO_ACTIONS"}) + typing_transactions = 0 + if journal.exists(): + for line in journal.read_text(encoding="utf-8").splitlines(): + try: + payload = json.loads(line) + except (TypeError, ValueError, json.JSONDecodeError): + continue + event = dict(payload.get("event") or payload) + if event.get("kind") == "typing_activity": + typing_transactions += 1 + fill_count = sum( + 1 for action in actions if action.get("action_type") in {"fill", "type"} + ) + if typing_transactions > fill_count: + issues.append( + { + "severity": "warning", + "code": "TEXT_INPUT_TRANSACTION_MISSING", + "typing_transactions": typing_transactions, + "fill_actions": fill_count, + } + ) + screenshot_count = len(list((root / "observations").glob("*/screen.png"))) + return { + "schema": "openclaw.ui_inspector.validation.v1", + "flow_dir": str(root), + "action_count": len(actions), + "element_count": len(elements), + "observation_count": len(list((root / "observations").glob("*/meta.json"))), + "screenshot_count": screenshot_count, + "error_count": sum(1 for issue in issues if issue["severity"] == "error"), + "warning_count": sum(1 for issue in issues if issue["severity"] == "warning"), + "issues": issues, + } diff --git a/tools/ui_inspector/diagnostics/replay_preflight.py b/tools/ui_inspector/diagnostics/replay_preflight.py new file mode 100644 index 0000000..3e5ca40 --- /dev/null +++ b/tools/ui_inspector/diagnostics/replay_preflight.py @@ -0,0 +1,133 @@ +"""Static, non-mutating replay readiness report for a finished capture.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +_INTERACTIVE = {"click", "fill", "type", "press_key", "select", "toggle"} + + +def _read(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def build_replay_preflight(flow_dir: Path) -> dict[str, Any]: + """Assess whether recorded actions contain executable, verified locators. + + This deliberately does not click or navigate. `event_time_unique` means + the locator count was measured in the event-origin document during capture; + it is not represented as a cross-session replay result. + """ + root = Path(flow_dir) + actions = sorted( + (_read(path) for path in (root / "actions").glob("*.json")), + key=lambda item: int(item.get("sequence") or 0), + ) + elements = { + item.get("id"): item + for item in (_read(path) for path in (root / "elements").glob("*.json")) + if item.get("id") + } + rows: list[dict[str, Any]] = [] + for action in actions: + action_type = str(action.get("action_type") or "") + if action_type not in _INTERACTIVE: + rows.append({ + "action_id": action.get("id"), + "sequence": action.get("sequence"), + "action_type": action_type, + "result": "context_only", + "reason": "non_interactive_or_navigation", + }) + continue + element_id = action.get("target_element_id") + element = elements.get(element_id) or {} + primary = dict(element.get("primary_locator") or {}) + eligible = bool(primary.get("production_eligible")) + unique = primary.get("unique") is True + score = float(primary.get("stability_score") or 0.0) + result = "pass" if eligible and unique and score >= 0.55 else "warning" + if not eligible or not unique: + result = "fail" + target = dict(action.get("target_snapshot") or {}) + bounds = dict(target.get("bounds") or target.get("rect") or {}) + area = float(bounds.get("width") or 0) * float(bounds.get("height") or 0) + oversized_click = action_type == "click" and area > 1_000_000 + root_window_target = ( + action_type == "click" + and str(target.get("control_type") or "") == "WindowControl" + ) + if oversized_click or root_window_target: + result = "fail" + validation = str(primary.get("validation") or "none") + verification = ( + "event_time_unique" if validation == "live" and unique + else "snapshot_unique" if validation in {"snapshot_only", "live_tree"} and unique + else "not_validated" + ) + raw_meta = dict((action.get("raw_event") or {}).get("meta") or {}) + rows.append({ + "action_id": action.get("id"), + "sequence": action.get("sequence"), + "action_type": action_type, + "result": result, + "element_id": element_id, + "logical_name": element.get("logical_name") or "", + "locator": primary.get("expression") or "", + "strategy": primary.get("strategy") or "", + "stability_score": score, + "match_count": primary.get("actual_count"), + "verification": verification, + "page_id": raw_meta.get("page_id") or "", + "document_id": raw_meta.get("document_id") or "", + "raw_event_ids": list(action.get("raw_event_ids") or []), + "reason": ( + "oversized_or_root_window_click_target" + if oversized_click or root_window_target else "" + ), + }) + counts = { + name: sum(1 for row in rows if row.get("result") == name) + for name in ("pass", "warning", "fail", "context_only") + } + from diagnostics.capture_validator import validate_capture + + quality = validate_capture(root) + blocking_codes = { + "TEXT_INPUT_TRANSACTION_MISSING", + "CLICK_TARGET_GENERIC_CONTAINER", + "KEY_TARGET_ROOT_WINDOW", + "CLICK_TARGET_OVERSIZED", + "ACTION_TARGET_MISSING", + "ZERO_ACTIONS", + } + quality_blockers = [ + issue + for issue in quality.get("issues") or [] + if issue.get("severity") == "error" or issue.get("code") in blocking_codes + ] + counts["quality_blockers"] = len(quality_blockers) + return { + "schema": "openclaw.ui_inspector.replay_preflight.v1", + "mode": "static_non_mutating", + "note": "事件时刻唯一性不等同于跨会话真实回放；本报告不会点击、输入或导航。", + "flow_dir": str(root), + "ready": not quality_blockers and counts.get("fail", 0) == 0, + "summary": counts, + "quality_gate": { + "passed": not quality_blockers, + "issues": quality_blockers, + }, + "actions": rows, + } + + +def write_replay_preflight(flow_dir: Path) -> dict[str, Any]: + report = build_replay_preflight(flow_dir) + path = Path(flow_dir) / "diagnostics" / "replay_report.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return report diff --git a/tools/ui_inspector/drivers/__init__.py b/tools/ui_inspector/drivers/__init__.py new file mode 100644 index 0000000..c1f7697 --- /dev/null +++ b/tools/ui_inspector/drivers/__init__.py @@ -0,0 +1,29 @@ +"""Surface drivers for UI Inspector (windows / browser / mobile stub).""" + +from __future__ import annotations + +from .base import DumpResult, SurfaceDriver +from .browser_pw import BrowserPlaywrightDriver +from .mobile import MobileDriver +from .windows_uia import WindowsUiaDriver +import target as _target + + +def get_driver(surface: str) -> SurfaceDriver: + if surface == _target.SURFACE_WINDOWS: + return WindowsUiaDriver() + if surface == _target.SURFACE_BROWSER: + return BrowserPlaywrightDriver() + if surface in {_target.SURFACE_ANDROID, _target.SURFACE_IOS}: + return MobileDriver(surface=surface) + raise RuntimeError(f"ERROR:SURFACE_UNSUPPORTED unknown surface={surface!r}") + + +__all__ = [ + "DumpResult", + "SurfaceDriver", + "WindowsUiaDriver", + "BrowserPlaywrightDriver", + "MobileDriver", + "get_driver", +] diff --git a/tools/ui_inspector/drivers/base.py b/tools/ui_inspector/drivers/base.py new file mode 100644 index 0000000..6898720 --- /dev/null +++ b/tools/ui_inspector/drivers/base.py @@ -0,0 +1,38 @@ +"""Driver protocol: dump one step's UI tree for a surface.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Optional, Protocol + + +@dataclass +class DumpResult: + surface: str + driver: str + window: dict[str, Any] + tree: dict[str, Any] + index: list[dict[str, Any]] + locatable: list[dict[str, Any]] + stats: dict[str, Any] + selection: dict[str, Any] = field(default_factory=dict) + url: str = "" + page_title: str = "" + locatable_text: str = "" + tree_text: str = "" + screenshot_bytes: Optional[bytes] = None # if set, artifacts write screen.png from bytes + use_hwnd_screenshot: bool = True # windows: ImageGrab by hwnd + + +class SurfaceDriver(Protocol): + def dump( + self, + *, + info: Any, + selection_meta: dict[str, Any], + max_depth: int = 16, + max_nodes: int = 8000, + warmup_mode: str = "full", + merge_same_pid_if_sparse: bool = True, + warmed_pids: Optional[set[int]] = None, + ) -> DumpResult: ... diff --git a/tools/ui_inspector/drivers/browser_pw.py b/tools/ui_inspector/drivers/browser_pw.py new file mode 100644 index 0000000..2f677bb --- /dev/null +++ b/tools/ui_inspector/drivers/browser_pw.py @@ -0,0 +1,90 @@ +"""Browser whole-page dump via Inspector Playwright session + kit page_dump.""" + +from __future__ import annotations + +from typing import Any, Optional + +import kit_browser +import target +from .base import DumpResult + + +class BrowserPlaywrightDriver: + surface = target.SURFACE_BROWSER + driver_name = "playwright" + + def dump( + self, + *, + info: Any, + selection_meta: dict[str, Any], + max_depth: int = 16, + max_nodes: int = 8000, + warmup_mode: str = "full", + merge_same_pid_if_sparse: bool = True, + warmed_pids: Optional[set[int]] = None, + page_id: str = "", + document_id: str = "", + frame_id: str = "", + ) -> DumpResult: + del max_depth, warmup_mode, merge_same_pid_if_sparse, warmed_pids # unused + sess = kit_browser.ensure_session() + dump = kit_browser.dump_current_page( + max_nodes=max_nodes, locatable_limit=300, page_id=page_id + ) + try: + png = kit_browser.screenshot_png(page_id=page_id) + except Exception: + png = None + + from jiangchang_skill_core.rpa.page_dump import ( + format_locatable_text, + format_tree_text, + ) + + url = dump.get("url") or "" + page_title = dump.get("title") or "" + window = info.to_dict() if hasattr(info, "to_dict") else dict(info or {}) + # Prefer live page identity for labeling / meta + window = { + **window, + "title": page_title or window.get("title") or "", + "class_name": window.get("class_name") or "PlaywrightPage", + "process_name": window.get("process_name") or "chrome.exe", + "url": url, + } + stats = dict(dump.get("stats") or {}) + active_page = dict(stats.get("active_page") or {}) + stats["browser_session"] = { + "profile_dir": sess.get("profile_dir"), + "reused": sess.get("reused"), + "page_count": active_page.get("page_count") or sess.get("page_count"), + "resolve": active_page.get("resolve") or "visible", + "note": ( + "dump 来自 Inspector 自管 Playwright 的当前可见标签页" + "（document.visibilityState），不是系统 Chrome 的 UIA，" + "也不是固定的 pages[0]。多标签时请切到目标页再拾取。" + ), + } + sel = dict(selection_meta or {}) + sel["surface"] = self.surface + sel["browser_url"] = url + sel["page_id"] = page_id or active_page.get("page_id") or "" + sel["document_id"] = document_id + sel["frame_id"] = frame_id + return DumpResult( + surface=self.surface, + driver=self.driver_name, + window=window, + tree=dump.get("tree") or {}, + index=list(dump.get("index") or []), + locatable=list(dump.get("locatable") or []), + stats=stats, + selection=sel, + url=url, + page_title=page_title, + locatable_text=format_locatable_text(dump.get("locatable") or []), + tree_text=format_tree_text(dump.get("tree") or {}), + screenshot_bytes=png, + use_hwnd_screenshot=False, + ) diff --git a/tools/ui_inspector/drivers/mobile.py b/tools/ui_inspector/drivers/mobile.py new file mode 100644 index 0000000..10145c0 --- /dev/null +++ b/tools/ui_inspector/drivers/mobile.py @@ -0,0 +1,30 @@ +"""Mobile surface stub — reserved for future android/ios dump.""" + +from __future__ import annotations + +from typing import Any, Optional + +import target +from .base import DumpResult + + +class MobileDriver: + def __init__(self, *, surface: str = target.SURFACE_ANDROID) -> None: + self.surface = surface + self.driver_name = "mobile_stub" + + def dump( + self, + *, + info: Any, + selection_meta: dict[str, Any], + max_depth: int = 16, + max_nodes: int = 8000, + warmup_mode: str = "full", + merge_same_pid_if_sparse: bool = True, + warmed_pids: Optional[set[int]] = None, + ) -> DumpResult: + raise RuntimeError( + f"ERROR:SURFACE_UNSUPPORTED 手机端拾取尚未实现（surface={self.surface}）。" + "请先用桌面或浏览器目标。" + ) diff --git a/tools/ui_inspector/drivers/windows_uia.py b/tools/ui_inspector/drivers/windows_uia.py new file mode 100644 index 0000000..f27f5f0 --- /dev/null +++ b/tools/ui_inspector/drivers/windows_uia.py @@ -0,0 +1,58 @@ +"""Windows desktop dump via existing desktop_uia.""" + +from __future__ import annotations + +from typing import Any, Optional + +import desktop_uia as uia +import target +from .base import DumpResult + + +class WindowsUiaDriver: + surface = target.SURFACE_WINDOWS + driver_name = "uiautomation" + + def dump( + self, + *, + info: Any, + selection_meta: dict[str, Any], + max_depth: int = 16, + max_nodes: int = 8000, + warmup_mode: str = "full", + merge_same_pid_if_sparse: bool = True, + warmed_pids: Optional[set[object]] = None, + ) -> DumpResult: + warmed = warmed_pids if warmed_pids is not None else set() + mode = warmup_mode + warm_key = (int(info.pid), str(info.class_name or "")) + if (warm_key in warmed or int(info.pid) in warmed) and mode == "full": + mode = "light" + merge_sparse = merge_same_pid_if_sparse + if (info.class_name or "") == "SearchResultWindow2": + merge_sparse = False + tree, index, stats, locatable = uia.dump_window_tree( + info.hwnd, + max_depth=max_depth, + max_nodes=max_nodes, + only_visible=False, + warmup_mode=mode, + merge_same_pid_if_sparse=merge_sparse, + ) + return DumpResult( + surface=self.surface, + driver=self.driver_name, + window=info.to_dict(), + tree=tree, + index=index, + locatable=locatable, + stats=stats, + selection=dict(selection_meta or {}), + url="", + page_title="", + locatable_text=uia.format_locatable_text(locatable), + tree_text=uia.format_tree_text(tree), + screenshot_bytes=None, + use_hwnd_screenshot=True, + ) diff --git a/tools/ui_inspector/evidence/__init__.py b/tools/ui_inspector/evidence/__init__.py new file mode 100644 index 0000000..ca5e14b --- /dev/null +++ b/tools/ui_inspector/evidence/__init__.py @@ -0,0 +1,6 @@ +"""Asynchronous evidence capture.""" + +from .coordinator import EvidenceCoordinator + +__all__ = ["EvidenceCoordinator"] + diff --git a/tools/ui_inspector/evidence/coordinator.py b/tools/ui_inspector/evidence/coordinator.py new file mode 100644 index 0000000..7800023 --- /dev/null +++ b/tools/ui_inspector/evidence/coordinator.py @@ -0,0 +1,255 @@ +"""Coalesced background full-page evidence capture. + +Semantic actions are committed immediately. Slow dumps are supplementary +evidence and are explicitly labelled eventual instead of being misrepresented +as an event-time after-state. +""" + +from __future__ import annotations + +import queue +import threading +import time +from dataclasses import dataclass +from typing import Any, Callable, Optional + + +@dataclass +class EvidenceRequest: + action_id: str + context: dict[str, Any] + requested_monotonic: float + + +class EvidenceCoordinator: + def __init__( + self, + manager: Any, + dump_fn: Callable[..., Any], + *, + coalesce_sec: float = 0.25, + on_dump: Optional[Callable[[Any], None]] = None, + ) -> None: + self.manager = manager + self.dump_fn = dump_fn + self.coalesce_sec = coalesce_sec + self.on_dump = on_dump + self._queue: queue.Queue[EvidenceRequest | None] = queue.Queue(maxsize=512) + self._urgent_queue: queue.Queue[EvidenceRequest | None] = queue.Queue(maxsize=128) + self._thread: Optional[threading.Thread] = None + self._urgent_thread: Optional[threading.Thread] = None + self._stop = threading.Event() + self.dropped = 0 + self.capture_count = 0 + self.failure_count = 0 + + def start(self) -> None: + if self._thread and self._thread.is_alive(): + return + self._stop.clear() + self._thread = threading.Thread(target=self._run, name="ui-inspector-evidence", daemon=True) + self._thread.start() + if str(getattr(self.manager.action_flow, "surface", "")) == "windows": + self._urgent_thread = threading.Thread( + target=self._run_urgent, + name="ui-inspector-evidence-transient", + daemon=True, + ) + self._urgent_thread.start() + + def schedule(self, action: Any) -> None: + raw = dict(getattr(action, "raw_event", {}) or {}) + meta = dict(raw.get("meta") or {}) + context = { + "page_id": meta.get("page_id") or "", + "document_id": meta.get("document_id") or "", + "frame_id": meta.get("frame_id") or "", + "source_ts_ms": meta.get("source_ts_ms"), + "url": raw.get("url") or "", + } + scope = dict((raw.get("target") or {}).get("window_scope") or {}) + context.update( + { + "target_hwnd": int(scope.get("hwnd") or 0), + "target_pid": int(scope.get("pid") or 0), + "target_class_name": str(scope.get("class_name") or ""), + "target_title": str(scope.get("title") or ""), + } + ) + out_queue = ( + self._urgent_queue + if context["target_class_name"] == "SearchResultWindow2" + else self._queue + ) + try: + out_queue.put_nowait(EvidenceRequest(action.id, context, time.monotonic())) + except queue.Full: + self.dropped += 1 + + def prime(self) -> None: + """Start one background enrichment without creating a semantic action.""" + try: + self._queue.put_nowait(EvidenceRequest("", {}, time.monotonic())) + except queue.Full: + self.dropped += 1 + + def _dump(self, context: dict[str, Any]) -> Any: + kwargs = { + k: context.get(k) + for k in ( + "page_id", "document_id", "frame_id", "target_hwnd", + "target_pid", "target_class_name", "target_title", + ) + if context.get(k) + } + if kwargs: + try: + return self.dump_fn(**kwargs) + except TypeError as exc: + if "unexpected keyword" not in str(exc) and "positional" not in str(exc): + raise + return self.dump_fn() + + def _run(self) -> None: + while not self._stop.is_set(): + try: + first = self._queue.get(timeout=0.2) + except queue.Empty: + continue + if first is None: + self._queue.task_done() + self._stop.set() + break + batch = [first] + stop_after_batch = False + deadline = time.monotonic() + self.coalesce_sec + while time.monotonic() < deadline: + try: + item = self._queue.get(timeout=max(0.01, deadline - time.monotonic())) + except queue.Empty: + break + if item is None: + self._queue.task_done() + stop_after_batch = True + break + batch.append(item) + + groups: dict[str, list[EvidenceRequest]] = {} + for item in batch: + key = str(item.context.get("page_id") or "active") + groups.setdefault(key, []).append(item) + for items in groups.values(): + latest = items[-1] + try: + dump = self._dump(latest.context) + if self.on_dump is not None: + self.on_dump(dump) + self.capture_count += 1 + # A desktop provider dump can take seconds. Actions that + # arrived while it was running are still earlier than this + # eventual observation, so fold them into the same result + # instead of immediately launching another identical dump. + if str(getattr(self.manager.action_flow, "surface", "")) == "windows": + while True: + try: + late = self._queue.get_nowait() + except queue.Empty: + break + if late is None: + self._queue.task_done() + stop_after_batch = True + break + batch.append(late) + items.append(late) + if not self._stop.is_set(): + self.manager.attach_eventual_evidence( + [item.action_id for item in items], dump=dump, + requested_monotonic=min(item.requested_monotonic for item in items), + ) + except Exception as exc: + self.failure_count += 1 + if not self._stop.is_set(): + self.manager.note_evidence_failure([item.action_id for item in items], str(exc)) + for _item in batch: + self._queue.task_done() + if stop_after_batch: + self._stop.set() + + def _run_urgent(self) -> None: + """Capture transient native popups independently of a slow main dump.""" + while not self._stop.is_set(): + try: + first = self._urgent_queue.get(timeout=0.1) + except queue.Empty: + continue + if first is None: + self._urgent_queue.task_done() + break + batch = [first] + # A very short coalesce keeps double-clicks together without waiting + # long enough for a search-result popup to disappear. + deadline = time.monotonic() + 0.04 + while time.monotonic() < deadline: + try: + item = self._urgent_queue.get_nowait() + except queue.Empty: + time.sleep(0.005) + continue + if item is None: + self._urgent_queue.task_done() + break + if item.context.get("target_hwnd") == first.context.get("target_hwnd"): + batch.append(item) + else: + # Preserve a different transient window for the next pass. + try: + self._urgent_queue.put_nowait(item) + except queue.Full: + self.dropped += 1 + self._urgent_queue.task_done() + break + try: + dump = self._dump(batch[-1].context) + if self.on_dump is not None: + self.on_dump(dump) + self.capture_count += 1 + if not self._stop.is_set(): + self.manager.attach_eventual_evidence( + [item.action_id for item in batch], + dump=dump, + requested_monotonic=min(item.requested_monotonic for item in batch), + ) + except Exception as exc: + self.failure_count += 1 + if not self._stop.is_set(): + self.manager.note_evidence_failure( + [item.action_id for item in batch], str(exc) + ) + finally: + for _item in batch: + self._urgent_queue.task_done() + + def stop(self, *, timeout: float = 35.0) -> None: + if not self._thread: + return + # Drain the transient-window lane first. A WeCom main-window dump can + # be stuck in an uninterruptible provider call for tens of seconds; it + # must not cancel the short-lived popup evidence when the user ends the + # flow immediately after selecting a result. + if self._urgent_thread: + try: + self._urgent_queue.put(None, timeout=1.0) + except queue.Full: + pass + self._urgent_thread.join(timeout=min(max(timeout, 3.0), 8.0)) + try: + self._queue.put(None, timeout=2.0) + except queue.Full: + pass + self._thread.join(timeout=timeout) + if self._thread.is_alive(): + # A provider dump may be uninterruptible. Prevent a late result + # from mutating a manager which is already being finalized. + self._stop.set() + self._thread = None + self._urgent_thread = None diff --git a/tools/ui_inspector/flow_session.py b/tools/ui_inspector/flow_session.py new file mode 100644 index 0000000..c37766d --- /dev/null +++ b/tools/ui_inspector/flow_session.py @@ -0,0 +1,408 @@ +"""Flow session glue: v2 ActionRecordingManager + surface recorders + v1 dual-write.""" + +from __future__ import annotations + +import queue +import threading +import uuid +from pathlib import Path +from typing import Any, Callable, Optional + +import artifacts as art +from core.pipeline.event_journal import EventJournal +from evidence.coordinator import EvidenceCoordinator +from recorder.base import RawUserEvent +from recorder.manager import ActionRecordingManager + + +class FlowSession: + def __init__( + self, + flow_dir: Path, + *, + flow_name: str, + surface: str, + recording_mode: str = "auto", + dump_fn: Optional[Callable[[], Any]] = None, + write_v1_step: Optional[Callable[..., Any]] = None, + exclude_hwnds: Optional[Callable[[], set[int]]] = None, + on_action_update: Optional[Callable[[Any, str], None]] = None, + stability_timeout_sec: float = 3.0, + ) -> None: + self.flow_dir = Path(flow_dir) + self.surface = surface + self.recording_mode = recording_mode + self._dump_fn = dump_fn + self.exclude_hwnds = exclude_hwnds or (lambda: set()) + self.manager = ActionRecordingManager( + self.flow_dir, + flow_name=flow_name, + flow_id=self.flow_dir.name, + surface=surface, + recording_mode=recording_mode, + dump_fn=dump_fn, + write_v1_step=write_v1_step, + stability_timeout_sec=stability_timeout_sec, + defer_action_evidence=( + surface in {"browser", "windows"} + and recording_mode != "legacy_interval" + ), + on_action_update=on_action_update, + ) + self.journal = EventJournal(self.flow_dir) + self._evidence = ( + EvidenceCoordinator(self.manager, dump_fn, on_dump=self._on_evidence_dump) + if surface in {"browser", "windows"} + and dump_fn is not None + and recording_mode != "legacy_interval" + else None + ) + if self._evidence is not None: + self.manager.on_action = self._evidence.schedule + self._recorder = None + self._recorder_health: dict[str, Any] = {} + self._event_queue: queue.Queue[tuple[str, Any]] = queue.Queue(maxsize=512) + self._event_thread: Optional[threading.Thread] = None + self._event_stop = threading.Event() + self._stats_lock = threading.Lock() + self._stats_cache: dict[str, Any] = { + "recorder_status": "idle", + "surface_recorder": "off", + "state_count": 0, + "observation_count": 0, + "action_count": 0, + "element_count": 0, + "queued_events": 0, + "dropped_events": 0, + } + self._dropped_events = 0 + + def start(self, *, entry_dump: Optional[Any] = None) -> None: + if self.recording_mode == "legacy_interval": + self.manager.start(entry_dump=entry_dump) + self._refresh_stats() + return + # Start observation before entry persistence. Events are journaled and + # buffered until manager.start() completes, eliminating startup loss. + self._start_surface_recorder() + if self.surface == "windows" and entry_dump is not None: + self._on_evidence_dump(entry_dump) + if self._evidence is not None: + self._evidence.start() + try: + self.manager.start(entry_dump=entry_dump) + except Exception: + if self._recorder: + self._recorder.stop() + self._recorder = None + if self._evidence is not None: + self._evidence.stop(timeout=3.0) + raise + self._start_event_consumer() + if self.surface == "windows" and self._evidence is not None: + # Warm/enrich UIA immediately, but on the evidence worker. Raw + # recording and Action reduction remain fully independent. + self._evidence.prime() + self._refresh_stats() + + def _submit(self, kind: str, payload: Any = None) -> bool: + try: + if kind == "stop": + self._event_queue.put((kind, payload), timeout=2.0) + else: + self._event_queue.put_nowait((kind, payload)) + return True + except queue.Full: + self._dropped_events += 1 + self.manager.last_warning = ( + f"recorder queue full; dropped={self._dropped_events}" + ) + self._refresh_stats() + return False + + def _start_event_consumer(self) -> None: + if self._event_thread and self._event_thread.is_alive(): + return + self._event_stop.clear() + + def consume() -> None: + while not self._event_stop.is_set(): + try: + kind, payload = self._event_queue.get(timeout=0.2) + except queue.Empty: + continue + try: + if kind == "stop": + break + if kind == "event": + self.manager.handle_raw_event(payload) + elif kind == "heartbeat": + self.manager.heartbeat() + elif kind == "pause": + self.manager.pause() + elif kind == "resume": + self.manager.resume() + except Exception as exc: + self.manager.last_warning = str(exc) + finally: + self._event_queue.task_done() + self._refresh_stats() + self._event_stop.set() + + self._event_thread = threading.Thread( + target=consume, + name="ui-inspector-action-consumer", + daemon=True, + ) + self._event_thread.start() + + def _start_surface_recorder(self) -> None: + def on_event(ev: RawUserEvent) -> None: + # Preserve the observed order and bound memory/thread growth. + if self.surface == "windows" and not (ev.meta or {}).get("event_id"): + ev.meta["event_id"] = f"windows:{uuid.uuid4().hex}" + self.journal.append(ev, channel="semantic") + self._submit("event", ev) + + def on_raw_event(payload: dict[str, Any]) -> None: + self.journal.append(payload, channel="browser_dom") + + if self.surface == "browser": + from surfaces.browser import BrowserRecorder + + self._recorder = BrowserRecorder( + on_event=on_event, on_raw_event=on_raw_event + ) + else: + from surfaces.windows import WindowsRecorder + + self._recorder = WindowsRecorder( + on_event=on_event, + exclude_hwnds=self.exclude_hwnds, + ) + self._seed_windows_element_atlas() + self._recorder.start() + + def _on_evidence_dump(self, dump: Any) -> None: + recorder = self._recorder + if self.surface != "windows" or recorder is None or not hasattr(recorder, "learn_elements"): + return + try: + recorder.learn_elements( + list(getattr(dump, "locatable", None) or []), + window=dict(getattr(dump, "window", None) or {}), + ) + except Exception: + pass + + def _seed_windows_element_atlas(self) -> None: + """Reuse recent captures as the event-time element repository. + + Values are discarded by ``learn_elements``; only stable names and + geometry are retained. This lets a new recording identify custom + editors before their slow accessibility provider completes a new dump. + """ + recorder = self._recorder + if recorder is None or not hasattr(recorder, "learn_elements"): + return + try: + siblings = sorted( + ( + path for path in self.flow_dir.parent.iterdir() + if path.is_dir() and path != self.flow_dir + ), + key=lambda path: path.stat().st_mtime, + reverse=True, + )[:8] + except Exception: + return + learned_classes: set[str] = set() + for sibling in siblings: + observations = sibling / "observations" + if not observations.is_dir(): + continue + try: + metas = sorted( + observations.glob("*/meta.json"), + key=lambda path: path.stat().st_mtime, + reverse=True, + ) + except Exception: + continue + for meta_path in metas: + try: + import json + + meta = json.loads(meta_path.read_text(encoding="utf-8")) + if str(meta.get("surface") or "") != "windows": + continue + selection = dict(meta.get("selection") or {}) + class_name = str( + selection.get("chosen_class") + or selection.get("event_class") + or "" + ) + if not class_name or class_name in learned_classes: + continue + locatable_path = meta_path.parent / "locatable.json" + rows = json.loads(locatable_path.read_text(encoding="utf-8")) + recorder.learn_elements( + rows, + window={ + "class_name": class_name, + "hwnd": int(selection.get("chosen_hwnd") or 0), + "pid": int(selection.get("chosen_pid") or 0), + "title": str(selection.get("event_title") or ""), + }, + ) + learned_classes.add(class_name) + except Exception: + continue + if "WeWorkWindow" in learned_classes and "SearchResultWindow2" in learned_classes: + break + + def pause(self) -> None: + if self._recorder: + self._recorder.pause() + self._submit("pause") + + def resume(self) -> None: + if self._recorder: + self._recorder.resume() + self._submit("resume") + + def stop(self) -> None: + recorder_stop_error: Optional[BaseException] = None + if self._recorder: + try: + self._recorder.stop() + except Exception as exc: + recorder_stop_error = exc + if hasattr(self._recorder, "health"): + try: + self._recorder_health = self._recorder.health() + except Exception as exc: + self._recorder_health = {"last_error": str(exc)} + self._recorder = None + if self._event_thread and self._event_thread.is_alive(): + self._submit("stop") + self._event_thread.join(timeout=35.0) + if self._event_thread.is_alive(): + self.manager.last_warning = "event consumer did not stop within 35s" + raise RuntimeError(self.manager.last_warning) + else: + self._event_thread = None + self._event_stop.set() + if self._evidence is not None: + self._evidence.stop(timeout=3.0) + self.manager.stop() + # Point flow.json v2 fields (already present from create_flow). + try: + flow = art.load_flow(self.flow_dir) + v2 = dict(flow.get("v2") or {}) + v2["action_count"] = len(self.manager.action_flow.actions) + v2["state_count"] = len(self.manager.action_flow.states) + v2["observation_count"] = len( + self.manager.snapshots.observation_ids() + ) + v2["element_count"] = len(self.manager.catalog.catalog.elements) + v2["raw_event_journal"] = "journal/raw_events.jsonl" + v2["pages"] = "pages.json" + v2["diagnostics_dir"] = "diagnostics/" + v2["validation"] = "diagnostics/validation.json" + v2["replay_report"] = "diagnostics/replay_report.json" + flow["v2"] = v2 + art.save_flow(self.flow_dir, flow) + except Exception: + pass + self._write_diagnostics() + self._refresh_stats() + if recorder_stop_error is not None: + self.manager.last_warning = str(recorder_stop_error) + raise RuntimeError(str(recorder_stop_error)) from recorder_stop_error + + def checkpoint(self) -> Any: + return self.manager.record_checkpoint() + + def heartbeat(self) -> None: + self._submit("heartbeat") + + def stats(self) -> dict[str, Any]: + with self._stats_lock: + return dict(self._stats_cache) + + def _refresh_stats(self) -> None: + st = self.manager.stats() + st["surface_recorder"] = ( + self._recorder.status if self._recorder else "off" + ) + st["queued_events"] = self._event_queue.qsize() + st["dropped_events"] = self._dropped_events + if self._recorder is not None and hasattr(self._recorder, "health"): + try: + st["input_backend"] = self._recorder.health() + except Exception as exc: + st["input_backend"] = {"last_error": str(exc)} + elif self._recorder_health: + st["input_backend"] = dict(self._recorder_health) + if self._evidence is not None: + st["evidence_capture_count"] = self._evidence.capture_count + st["evidence_dropped"] = self._evidence.dropped + st["evidence_queued"] = self._evidence._queue.qsize() + st["transient_evidence_queued"] = self._evidence._urgent_queue.qsize() + st["evidence_failures"] = self._evidence.failure_count + with self._stats_lock: + self._stats_cache = st + + def _write_diagnostics(self) -> None: + pages: dict[str, dict[str, Any]] = {} + lags: list[int] = [] + for action in self.manager._actions_by_id.values(): + raw = dict(action.raw_event or {}) + meta = dict(raw.get("meta") or {}) + page_id = str(meta.get("page_id") or "") + if page_id: + page = pages.setdefault(page_id, {"page_id": page_id, "urls": [], "document_ids": []}) + url = str(raw.get("url") or (action.input or {}).get("url") or "") + document_id = str(meta.get("document_id") or "") + if url and url not in page["urls"]: + page["urls"].append(url) + if document_id and document_id not in page["document_ids"]: + page["document_ids"].append(document_id) + lag = (action.evidence or {}).get("evidence_lag_ms") + if isinstance(lag, int): + lags.append(lag) + diagnostics = self.flow_dir / "diagnostics" + diagnostics.mkdir(parents=True, exist_ok=True) + try: + art.write_json(self.flow_dir / "pages.json", { + "schema": "openclaw.ui_inspector.pages.v1", + "pages": list(pages.values()), + }) + recorder_health = dict(self._recorder_health) + if self._recorder is not None and hasattr(self._recorder, "health"): + try: + recorder_health = self._recorder.health() + except Exception as exc: + recorder_health = {"last_error": str(exc)} + art.write_json(diagnostics / "metrics.json", { + "schema": "openclaw.ui_inspector.metrics.v1", + "raw_event_count": self.journal._sequence, + "action_count": len(self.manager.action_flow.actions), + "state_count": len(self.manager.action_flow.states), + "observation_count": len(self.manager.snapshots.observation_ids()), + "element_count": len(self.manager.catalog.catalog.elements), + "event_queue_dropped": self._dropped_events, + "evidence_queue_dropped": self._evidence.dropped if self._evidence else 0, + "evidence_capture_count": self._evidence.capture_count if self._evidence else 0, + "evidence_failure_count": self._evidence.failure_count if self._evidence else 0, + "evidence_lag_ms_max": max(lags) if lags else None, + "evidence_lag_ms_average": round(sum(lags) / len(lags), 2) if lags else None, + "input_backend": recorder_health, + }) + from diagnostics import validate_capture, write_replay_preflight + + art.write_json(diagnostics / "validation.json", validate_capture(self.flow_dir)) + write_replay_preflight(self.flow_dir) + except Exception as exc: + self.manager.last_warning = f"diagnostics write failed: {exc}" diff --git a/tools/ui_inspector/kit_browser.py b/tools/ui_inspector/kit_browser.py new file mode 100644 index 0000000..b39df20 --- /dev/null +++ b/tools/ui_inspector/kit_browser.py @@ -0,0 +1,729 @@ +"""Inspector-owned Playwright browser session (dev-time page dump). + +All Playwright sync API calls run on one dedicated worker thread +(Playwright sync is not cross-thread safe). + +Dump / screenshot / recorder target the **currently visible** page +(document.visibilityState === 'visible'), not a sticky pages[0]. +This keeps list ↔ detail tab switching aligned with what the user sees. + +Uses jiangchang_skill_core.rpa stealth launch parts + dump_page_sync. +Profile lives under ui_inspector/_browser_profile (not account-manager). +""" + +from __future__ import annotations + +import atexit +import queue +import threading +import uuid +from pathlib import Path +from typing import Any, Callable, Optional + +_HERE = Path(__file__).resolve().parent +_DEFAULT_PROFILE = _HERE / "_browser_profile" +_DEFAULT_START_URL = "about:blank" + +_cmd_q: queue.Queue = queue.Queue() +_worker: Optional[threading.Thread] = None +_worker_lock = threading.Lock() + +# Owned exclusively by worker thread: +_pw = None +_context = None +_page = None +_started = False +_bridge_installed = False +_bridge_sink: Optional[Callable[[dict[str, Any]], None]] = None +_bridge_lock = threading.Lock() +_BRIDGE_NAME = "__openclawUiInspectorEmit" +_page_ids: dict[int, str] = {} +_page_refs: dict[int, Any] = {} +_pages_by_id: dict[str, Any] = {} +_frame_ids: dict[int, str] = {} +_frame_refs: dict[int, Any] = {} + + +def _page_id_for(page: Any) -> str: + if not _page_is_open(page): + return "" + key = id(page) + page_id = _page_ids.get(key) if _page_refs.get(key) is page else None + if not page_id: + page_id = f"page_{uuid.uuid4().hex[:12]}" + _page_ids[key] = page_id + _page_refs[key] = page + _pages_by_id[page_id] = page + return page_id + + +def _frame_id_for(frame: Any, *, page_id: str = "") -> str: + if frame is None: + return "" + key = id(frame) + frame_id = _frame_ids.get(key) if _frame_refs.get(key) is frame else None + if not frame_id: + prefix = page_id or "page" + frame_id = f"{prefix}:frame_{uuid.uuid4().hex[:10]}" + _frame_ids[key] = frame_id + _frame_refs[key] = frame + return frame_id + + +def _page_for_id(page_id: str) -> Any: + page = _pages_by_id.get(str(page_id or "")) + if _page_is_open(page): + return page + if page_id: + _pages_by_id.pop(str(page_id), None) + return None + + +def kit_page_dump_available() -> bool: + try: + from jiangchang_skill_core.rpa.page_dump import dump_page_sync # noqa: F401 + + return True + except ImportError: + return False + + +def profile_dir() -> Path: + return _DEFAULT_PROFILE + + +def _ensure_worker() -> None: + global _worker + with _worker_lock: + if _worker is not None and _worker.is_alive(): + return + + def loop() -> None: + while True: + item = _cmd_q.get() + if item is None: + break + fn, args, kwargs, out_q = item + try: + out_q.put(("ok", fn(*args, **kwargs))) + except BaseException as exc: + out_q.put(("err", exc)) + + _worker = threading.Thread(target=loop, name="ui-inspector-pw", daemon=True) + _worker.start() + atexit.register(shutdown) + + +def _call_on_worker( + fn: Callable[..., Any], + *args: Any, + timeout_sec: float = 30.0, + **kwargs: Any, +) -> Any: + _ensure_worker() + out_q: queue.Queue = queue.Queue() + _cmd_q.put((fn, args, kwargs, out_q)) + try: + status, payload = out_q.get(timeout=max(0.1, float(timeout_sec))) + except queue.Empty as exc: + raise RuntimeError( + f"ERROR:BROWSER_WORKER_TIMEOUT Playwright 命令超过 {timeout_sec:g} 秒未返回。" + ) from exc + if status == "err": + raise payload + return payload + + +def _teardown_unlocked() -> None: + global _pw, _context, _page, _started, _bridge_installed, _bridge_sink + try: + if _context is not None: + _context.close() + except Exception: + pass + try: + if _pw is not None: + _pw.stop() + except Exception: + pass + _pw = None + _context = None + _page = None + _started = False + _bridge_installed = False + with _bridge_lock: + _bridge_sink = None + _page_ids.clear() + _page_refs.clear() + _pages_by_id.clear() + _frame_ids.clear() + _frame_refs.clear() + + +def _goto_safe(page: Any, start_url: str) -> dict[str, Any]: + url = (start_url or "").strip() + if not url or url == "about:blank": + return {"ok": True, "requested_url": url, "actual_url": page.url or ""} + try: + page.goto(url, wait_until="domcontentloaded", timeout=60_000) + except Exception as exc: + actual = "" + try: + actual = str(page.url or "") + except Exception: + pass + # Some sites keep loading after domcontentloaded timeout. A real HTTP + # page is usable, but the warning must remain visible to callers. + if actual.startswith(("http://", "https://")): + return { + "ok": True, + "requested_url": url, + "actual_url": actual, + "warning": str(exc), + } + raise RuntimeError( + f"ERROR:NAVIGATION_FAILED requested={url!r} actual={actual!r}: {exc}" + ) from exc + actual = str(page.url or "") + if not actual or actual == "about:blank" or actual.startswith("about:"): + raise RuntimeError( + f"ERROR:NAVIGATION_FAILED requested={url!r} actual={actual!r}" + ) + return {"ok": True, "requested_url": url, "actual_url": actual} + + +def _page_is_open(page: Any) -> bool: + try: + return page is not None and not page.is_closed() + except Exception: + return False + + +def _open_pages(context: Any) -> list[Any]: + if context is None: + return [] + out: list[Any] = [] + try: + pages = list(context.pages or []) + except Exception: + return [] + for p in pages: + if _page_is_open(p): + _page_id_for(p) + out.append(p) + return out + + +def _page_visibility(page: Any) -> str: + """Return document.visibilityState or '' on failure.""" + if not _page_is_open(page): + return "" + try: + state = page.evaluate("() => document.visibilityState") + return str(state or "").strip().lower() + except Exception: + return "" + + +def resolve_active_page( + context: Any, + *, + preferred: Any = None, +) -> tuple[Any, dict[str, Any]]: + """Pick the page the user is looking at (visible tab). + + Returns (page, meta). Prefer visibilityState===visible; fall back to + preferred if still open, else first open page. + """ + pages = _open_pages(context) + meta: dict[str, Any] = { + "resolve": "none", + "page_count": len(pages), + "visible_count": 0, + } + if not pages: + raise RuntimeError("ERROR:NO_OPEN_PAGE Playwright 会话中没有可用标签页。") + + visible: list[Any] = [] + for p in pages: + if _page_visibility(p) == "visible": + visible.append(p) + meta["visible_count"] = len(visible) + + if len(visible) == 1: + meta["resolve"] = "visibility" + return visible[0], meta + if len(visible) > 1: + # Multiple windows each with a visible tab — prefer preferred if among them + if preferred is not None and preferred in visible: + meta["resolve"] = "visibility_preferred" + return preferred, meta + meta["resolve"] = "visibility_first" + return visible[0], meta + + if preferred is not None and _page_is_open(preferred) and preferred in pages: + meta["resolve"] = "preferred_fallback" + return preferred, meta + + meta["resolve"] = "first_open" + return pages[0], meta + + +def _sync_active_page(*, update_global: bool = True) -> tuple[Any, dict[str, Any]]: + """Resolve visible page; optionally assign to module ``_page``.""" + global _page + if _context is None: + raise RuntimeError("ERROR:NO_BROWSER_SESSION Playwright 会话未启动。") + page, meta = resolve_active_page(_context, preferred=_page) + if update_global: + _page = page + try: + meta["url"] = page.url or "" + meta["title"] = page.title() or "" + except Exception: + meta["url"] = "" + meta["title"] = "" + meta["page_id"] = _page_id_for(page) + return page, meta + + +def _resolve_capture_page(page_id: str = "") -> tuple[Any, dict[str, Any]]: + global _page + requested = str(page_id or "") + if requested: + page = _page_for_id(requested) + if page is not None: + _page = page + meta = { + "resolve": "event_page_id", + "page_id": requested, + "page_count": len(_open_pages(_context)), + "visible_count": None, + } + try: + meta["url"] = page.url or "" + meta["title"] = page.title() or "" + except Exception: + meta["url"] = "" + meta["title"] = "" + return page, meta + raise RuntimeError(f"ERROR:EVENT_PAGE_CLOSED page_id={requested}") + return _sync_active_page(update_global=True) + + +def _ensure_session_worker( + *, + start_url: str = _DEFAULT_START_URL, + force_goto: bool = False, +) -> dict[str, Any]: + global _pw, _context, _page, _started + want = (start_url or "").strip() or _DEFAULT_START_URL + + if _started and _context is not None: + try: + navigation = None + pages = _open_pages(_context) + if not pages: + _teardown_unlocked() + else: + # Start-flow navigation: prefer first open tab (stable home), + # not whatever detail tab happens to be visible. + if force_goto and want and want != "about:blank": + home = pages[0] + cur = "" + try: + cur = home.url or "" + except Exception: + cur = "" + if cur.rstrip("/") != want.rstrip("/"): + navigation = _goto_safe(home, want) + _page = home + else: + _page, _meta = _sync_active_page(update_global=True) + if want and want != "about:blank" and ( + (_page.url or "") in {"", "about:blank"} + or str(_page.url or "").startswith("about:") + ): + navigation = _goto_safe(_page, want) + _page, _meta = _sync_active_page(update_global=True) + try: + live_url = _page.url or "" + except Exception: + live_url = "" + return { + "ok": True, + "reused": True, + "url": live_url, + "profile_dir": str(profile_dir()), + "page_count": len(_open_pages(_context)), + "navigation": navigation, + } + except Exception: + _teardown_unlocked() + + if not kit_page_dump_available(): + raise RuntimeError( + "ERROR:KIT_PAGE_DUMP_MISSING 需要 jiangchang-platform-kit>=1.2.11 " + "(jiangchang_skill_core.rpa.page_dump)。请升级 kit 后重试。" + ) + + try: + from playwright.sync_api import sync_playwright + except ImportError as exc: + raise RuntimeError( + "ERROR:MISSING_PLAYWRIGHT 未安装 playwright。" + "请在与技能相同的 Python 环境中安装后重试。" + ) from exc + + from jiangchang_skill_core.runtime_env import find_chrome_executable + from jiangchang_skill_core.rpa.stealth import ( + STEALTH_INIT_SCRIPT, + persistent_context_launch_parts, + stealth_enabled, + ) + + chrome = find_chrome_executable() + if not chrome: + raise RuntimeError( + "ERROR:MISSING_BROWSER 未检测到 Chrome 或 Edge，请安装后重试。" + ) + + prof = profile_dir() + prof.mkdir(parents=True, exist_ok=True) + args, ignore = persistent_context_launch_parts(extra_args=["--start-maximized"]) + launch_kwargs: dict[str, Any] = dict( + user_data_dir=str(prof), + headless=False, + executable_path=chrome, + locale="zh-CN", + no_viewport=True, + args=args, + ) + if ignore is not None: + launch_kwargs["ignore_default_args"] = ignore + + _pw = sync_playwright().start() + try: + _context = _pw.chromium.launch_persistent_context(**launch_kwargs) + if stealth_enabled(): + _context.add_init_script(STEALTH_INIT_SCRIPT) + if _context.pages: + _page = _context.pages[0] + else: + _page = _context.new_page() + navigation = None + if want and want != "about:blank": + navigation = _goto_safe(_page, want) + elif want and ( + _page.url in {"", "about:blank"} or str(_page.url).startswith("about:") + ): + navigation = _goto_safe(_page, want) + _started = True + return { + "ok": True, + "reused": False, + "url": _page.url, + "profile_dir": str(prof), + "page_count": len(_open_pages(_context)), + "navigation": navigation, + } + except Exception: + _teardown_unlocked() + raise + + +def _dump_worker(*, max_nodes: int, locatable_limit: int, page_id: str = "") -> dict[str, Any]: + from jiangchang_skill_core.rpa.page_dump import dump_page_sync + + _ensure_session_worker() + page, meta = _resolve_capture_page(page_id) + result = dump_page_sync(page, max_nodes=max_nodes, locatable_limit=locatable_limit) + stats = dict(result.get("stats") or {}) + stats["active_page"] = meta + result["stats"] = stats + return result + + +def _screenshot_worker(*, page_id: str = "") -> bytes: + _ensure_session_worker() + page, _meta = _resolve_capture_page(page_id) + return page.screenshot(type="png", full_page=False) + + +def _session_info_worker() -> dict[str, Any]: + if not _started or _context is None: + return {"active": False} + try: + page, meta = _sync_active_page(update_global=True) + return { + "active": True, + "url": page.url, + "title": page.title(), + "profile_dir": str(profile_dir()), + "page_count": meta.get("page_count"), + "resolve": meta.get("resolve"), + "page_id": meta.get("page_id"), + } + except Exception as exc: + return {"active": False, "error": str(exc)} + + +def _dispatch_bridge_event(source: Any, payload: Any) -> None: + if not isinstance(payload, dict): + return + item = dict(payload) + try: + page = source.get("page") if isinstance(source, dict) else None + frame = source.get("frame") if isinstance(source, dict) else None + page_id = _page_id_for(page) + item["page_id"] = page_id + item["frame_id"] = _frame_id_for(frame, page_id=page_id) + except Exception: + pass + with _bridge_lock: + sink = _bridge_sink + if sink is not None: + try: + sink(item) + except Exception: + pass + + +def _install_event_bridge_worker( + *, init_script: str, sink: Callable[[dict[str, Any]], None] +) -> dict[str, Any]: + global _bridge_installed, _bridge_sink + _ensure_session_worker() + assert _context is not None + with _bridge_lock: + _bridge_sink = sink + if not _bridge_installed: + _context.expose_binding(_BRIDGE_NAME, _dispatch_bridge_event) + _bridge_installed = True + _context.add_init_script(init_script) + installed = 0 + for page in _open_pages(_context): + try: + page.evaluate(init_script) + installed += 1 + except Exception: + pass + return {"ok": True, "binding": _BRIDGE_NAME, "installed_pages": installed} + + +def install_event_bridge( + init_script: str, sink: Callable[[dict[str, Any]], None] +) -> dict[str, Any]: + """Install a context-level recorder bridge before user actions begin.""" + return _call_on_worker( + _install_event_bridge_worker, + init_script=init_script, + sink=sink, + timeout_sec=15.0, + ) + + +def clear_event_bridge() -> None: + global _bridge_sink + with _bridge_lock: + _bridge_sink = None + + +def ensure_session( + *, + start_url: str = _DEFAULT_START_URL, + force_goto: bool = False, +) -> dict[str, Any]: + """Start or reuse headed persistent context; return status dict. + + force_goto=True: navigate to start_url even when session is reused + (used when「开始流程」带起始 URL). + """ + return _call_on_worker( + _ensure_session_worker, + start_url=start_url, + force_goto=force_goto, + timeout_sec=75.0, + ) + + +def session_info() -> dict[str, Any]: + return _call_on_worker(_session_info_worker, timeout_sec=10.0) + + +def dump_current_page( + *, max_nodes: int = 8000, locatable_limit: int = 300, page_id: str = "" +) -> dict[str, Any]: + """Dump an event-origin page, or resolve the active page as fallback.""" + return _call_on_worker( + _dump_worker, + max_nodes=max_nodes, + locatable_limit=locatable_limit, + page_id=page_id, + timeout_sec=30.0, + ) + + +def screenshot_png(*, page_id: str = "") -> bytes: + return _call_on_worker(_screenshot_worker, page_id=page_id, timeout_sec=20.0) + + +def page_id_for(page: Any) -> str: + """Return the stable id for a Page while already on the browser worker.""" + return _page_id_for(page) + + +def _validate_element_worker( + *, page_id: str, element: dict[str, Any], frame: Optional[dict[str, Any]] = None +) -> dict[str, int]: + page, _meta = _resolve_capture_page(page_id) + scope: Any = page + frame_meta = dict(frame or {}) + frame_url = str(frame_meta.get("url") or "") + frame_name = str(frame_meta.get("name") or "") + if frame_url or frame_name: + try: + for candidate in page.frames: + if frame_name and candidate.name == frame_name: + scope = candidate + break + if frame_url and candidate.url == frame_url: + scope = candidate + break + except Exception: + pass + + el = dict(element or {}) + role = str(el.get("role") or "").strip() + name = str(el.get("name") or "").strip() + test_id = str(el.get("test_id") or "").strip() + label = str(el.get("label") or "").strip() + placeholder = str(el.get("placeholder") or "").strip() + name_attr = str(el.get("name_attr") or "").strip() + href = str(el.get("href") or "").strip() + text_value = str(el.get("text") or "").strip() + css_id = str(el.get("id") or "").strip() + css_path = str(el.get("css") or el.get("css_path") or "").strip() + stable_css = str(el.get("stable_css") or "").strip() + semantic_role = str(el.get("semantic_role") or "").strip() + semantic_name = str(el.get("semantic_name") or "").strip() + counts: dict[str, int] = {} + + def count(strategy: str, build: Callable[[], Any]) -> None: + try: + counts[strategy] = int(build().count()) + except Exception: + pass + + def count_actionable(strategy: str, build: Callable[[], Any]) -> None: + try: + locator = build() + total = min(int(locator.count()), 50) + actionable = 0 + for index in range(total): + item = locator.nth(index) + if item.is_visible() and item.is_enabled(): + actionable += 1 + counts[strategy] = actionable + except Exception: + pass + + if role and name: + # Playwright accessible names normalize whitespace (SVG/icon buttons are + # a common example), while DOM innerText may not. Non-exact still + # yields a trustworthy uniqueness count and mirrors getByRole defaults. + count("role", lambda: scope.get_by_role(role, name=name, exact=False)) + elif role and bool(el.get("contenteditable")): + count("role", lambda: scope.get_by_role(role)) + if semantic_role and semantic_name: + count_actionable( + "semantic_role", + lambda: scope.get_by_role( + semantic_role, name=semantic_name, exact=False + ), + ) + if test_id: + count("test_id", lambda: scope.get_by_test_id(test_id)) + if label: + count("label", lambda: scope.get_by_label(label, exact=True)) + if placeholder: + count("placeholder", lambda: scope.get_by_placeholder(placeholder, exact=True)) + if name_attr: + escaped_name = name_attr.replace("\\", "\\\\").replace('"', '\\"') + count("name_attr", lambda: scope.locator(f'[name="{escaped_name}"]')) + if href: + escaped_href = href.replace("\\", "\\\\").replace('"', '\\"') + count("href", lambda: scope.locator(f'a[href="{escaped_href}"]')) + if text_value and len(text_value) <= 80: + count("text", lambda: scope.get_by_text(text_value, exact=True)) + if css_id and not css_id[0].isdigit(): + escaped = css_id.replace("\\", "\\\\").replace('"', '\\"') + count("css_id", lambda: scope.locator(f'[id="{escaped}"]')) + if css_path: + count("css", lambda: scope.locator(css_path)) + if stable_css: + count("stable_css", lambda: scope.locator(stable_css)) + return counts + + +def validate_element_on_page( + *, page_id: str, element: dict[str, Any], frame: Optional[dict[str, Any]] = None +) -> dict[str, int]: + """Count locator candidates against the exact live event-origin page.""" + if not page_id: + return {} + return _call_on_worker( + _validate_element_worker, + page_id=page_id, + element=element, + frame=frame, + timeout_sec=10.0, + ) + + +def call_on_page(fn: Callable[[Any], Any]) -> Any: + """Run ``fn(page)`` on the **visible** Playwright page (worker thread).""" + + def _run() -> Any: + _ensure_session_worker() + page, _meta = _sync_active_page(update_global=True) + return fn(page) + + return _call_on_worker(_run, timeout_sec=30.0) + + +def call_on_each_open_page(fn: Callable[[Any], Any]) -> list[Any]: + """Run ``fn(page)`` for every open tab; returns list of per-page results.""" + + def _run() -> list[Any]: + _ensure_session_worker() + assert _context is not None + pages = _open_pages(_context) + # Keep _page synced to visible for dump consistency + try: + _sync_active_page(update_global=True) + except Exception: + pass + results: list[Any] = [] + for p in pages: + try: + results.append(fn(p)) + except Exception as exc: + results.append({"error": str(exc)}) + return results + + return _call_on_worker(_run, timeout_sec=20.0) + + +def shutdown() -> None: + def _stop() -> None: + _teardown_unlocked() + + try: + _call_on_worker(_stop, timeout_sec=5.0) + except Exception: + pass + try: + _cmd_q.put(None) + except Exception: + pass diff --git a/tools/ui_inspector/kit_wake.py b/tools/ui_inspector/kit_wake.py index e35c076..abbf0c6 100644 --- a/tools/ui_inspector/kit_wake.py +++ b/tools/ui_inspector/kit_wake.py @@ -3,6 +3,9 @@ - 主窗 WeWorkWindow → ``ensure_ui_ready("wecom")`` - 搜索浮层 SearchResultWindow2 → ``ensure_ui_ready("wecom_search")`` 未安装 kit 时降级为跳过，不阻断采集。 + +Warm cache key = profile + hwnd（经 WeCom Adapter），避免主窗 warm +状态错误代表 SearchResultWindow2 已暖。 """ from __future__ import annotations @@ -10,9 +13,13 @@ from __future__ import annotations import time from typing import Any, Optional -_CACHE_TTL_SEC = 45.0 +# A successful provider probe remains valid for the lifetime of a normal +# recording flow. The old 45 second TTL made long desktop flows re-run an +# expensive WeCom wake in the middle of recording and produced minute-late +# evidence. +_CACHE_TTL_SEC = 600.0 -# profile -> cache +# cache_key -> cache (profile:hwnd:N) _caches: dict[str, dict[str, Any]] = {} _PROFILE_WECOM = "wecom" @@ -24,8 +31,26 @@ _THRESHOLDS = { } +def _wecom_adapter(): + try: + from adapters.wecom import wecom_adapter + + return wecom_adapter + except Exception: + try: + from .adapters.wecom import wecom_adapter # type: ignore + + return wecom_adapter + except Exception: + return None + + def resolve_wake_profile(*, class_name: str = "", title: str = "") -> Optional[str]: """返回应调用的 kit profile id；非企微相关则 None。""" + ad = _wecom_adapter() + if ad is not None: + return ad.wake_profile(class_name=class_name, title=title) + # Compatibility fallback if adapter import fails cls = (class_name or "").strip() title = title or "" if cls == "SearchResultWindow2": @@ -50,6 +75,13 @@ def kit_available() -> bool: return False +def _cache_key(profile: str, hwnd: Optional[int]) -> str: + ad = _wecom_adapter() + if ad is not None: + return ad.cache_key(profile=profile, hwnd=int(hwnd or 0)) + return f"{profile}:hwnd:{int(hwnd or 0)}" + + def auto_ensure_if_wecom( *, class_name: str = "", @@ -68,17 +100,22 @@ def auto_ensure_if_wecom( "available": kit_available(), } - warm_min = int(_THRESHOLDS.get(profile, 80)) + ad = _wecom_adapter() + warm_min = int(ad.warm_threshold(profile) if ad else _THRESHOLDS.get(profile, 80)) + key = _cache_key(profile, hwnd) cache = _caches.setdefault( - profile, {"t": 0.0, "warm": False, "hwnd": 0, "findall": 0} + key, {"t": 0.0, "warm": False, "hwnd": int(hwnd or 0), "findall": 0, "profile": profile} ) now = time.monotonic() hint = int(findall_hint) if findall_hint is not None else -1 + # Require same hwnd in cache — main window warm must not cover search popup. + cached_hwnd = int(cache.get("hwnd") or 0) + same_window = (not hwnd) or (cached_hwnd == int(hwnd)) if ( not force + and same_window and cache.get("warm") and (now - float(cache.get("t") or 0.0)) < _CACHE_TTL_SEC - and (hint < 0 or hint >= warm_min) ): return { "applicable": True, @@ -91,6 +128,28 @@ def auto_ensure_if_wecom( "findall": int(cache.get("findall") or 0), "hwnd": int(cache.get("hwnd") or 0), "profile": profile, + "cache_key": key, + "observed_findall_hint": hint, + } + + # A direct probe is authoritative. Do not run an invasive wake merely + # because this process did not create the cache entry itself. + if not force and hint >= warm_min: + cache.update( + {"t": now, "warm": True, "hwnd": int(hwnd or 0), "findall": hint} + ) + return { + "applicable": True, + "available": kit_available(), + "skipped": True, + "reason": "probe_warm", + "ok": True, + "warm": True, + "wake_applied": False, + "findall": hint, + "hwnd": int(hwnd or 0), + "profile": profile, + "cache_key": key, } try: @@ -157,16 +216,49 @@ def auto_ensure_if_wecom( "profile": profile, "markers_found": list(after.markers_found) if after is not None else [], } + report["cache_key"] = key if result.warm: - cache["t"] = now - cache["warm"] = True - cache["hwnd"] = report["hwnd"] - cache["findall"] = findall + # Persist under result hwnd so main≠search caches stay separate. + final_key = _cache_key(profile, report["hwnd"] or hwnd) + final_cache = _caches.setdefault( + final_key, + {"t": 0.0, "warm": False, "hwnd": int(report["hwnd"] or 0), "findall": 0, "profile": profile}, + ) + final_cache["t"] = now + final_cache["warm"] = True + final_cache["hwnd"] = int(report["hwnd"] or hwnd or 0) + final_cache["findall"] = findall + report["cache_key"] = final_key else: cache["warm"] = False return report +def note_provider_warm( + *, class_name: str = "", title: str = "", hwnd: int = 0, findall: int = 0 +) -> bool: + """Pin a proven-warm provider so recorder evidence never re-wakes it.""" + profile = resolve_wake_profile(class_name=class_name, title=title) + adapter = _wecom_adapter() + threshold = int( + adapter.warm_threshold(profile) + if adapter and profile + else _THRESHOLDS.get(profile or "", 80) + ) + count = int(findall or 0) + if not profile or count < threshold: + return False + key = _cache_key(profile, hwnd) + _caches[key] = { + "t": time.monotonic(), + "warm": True, + "hwnd": int(hwnd or 0), + "findall": count, + "profile": profile, + } + return True + + def format_kit_wake_line(report: Optional[dict[str, Any]]) -> str: if not report: return "kit_wake: (none)" diff --git a/tools/ui_inspector/locators/__init__.py b/tools/ui_inspector/locators/__init__.py new file mode 100644 index 0000000..178105c --- /dev/null +++ b/tools/ui_inspector/locators/__init__.py @@ -0,0 +1,13 @@ +"""Locator compilers + validators + element catalog.""" + +from .browser import compile_browser_locators +from .catalog import ElementCatalogService +from .validator import validate_locator_against_rows +from .windows import compile_windows_locators + +__all__ = [ + "compile_browser_locators", + "compile_windows_locators", + "validate_locator_against_rows", + "ElementCatalogService", +] diff --git a/tools/ui_inspector/locators/base.py b/tools/ui_inspector/locators/base.py new file mode 100644 index 0000000..78f7517 --- /dev/null +++ b/tools/ui_inspector/locators/base.py @@ -0,0 +1,74 @@ +"""Shared locator compiler helpers.""" + +from __future__ import annotations + +from typing import Any, Optional + +from models.locator import LocatorCandidate + + +def rank_and_select_primary( + candidates: list[LocatorCandidate], +) -> tuple[Optional[LocatorCandidate], list[LocatorCandidate]]: + """ + Pick production primary only if unique and not coordinate-based. + Fallback list excludes primary. + """ + ranked = sorted( + candidates, + key=lambda c: ( + 0 if c.production_eligible else 1, + 0 if c.unique else 1, + 0 if c.uses_coordinates else 0, + 1 if c.uses_coordinates else 0, + 1 if c.uses_index else 0, + -float(c.stability_score or 0.0), + ), + ) + primary: Optional[LocatorCandidate] = None + for c in ranked: + if c.production_eligible and not c.uses_coordinates: + primary = c + break + fallbacks = [c for c in ranked if c is not primary] + return primary, fallbacks + + +def count_matches( + rows: list[dict[str, Any]], + *, + properties: dict[str, Any], + scope: Optional[dict[str, Any]] = None, +) -> int: + """Count rows matching structured properties (snapshot / live tree).""" + props = {k: v for k, v in (properties or {}).items() if v not in (None, "")} + if not props: + return 0 + scope = scope or {} + n = 0 + for r in rows or []: + if not isinstance(r, dict): + continue + if scope.get("window_class"): + # optional soft filter if row carries window info + rc = r.get("window_class") or r.get("window_class_name") + if rc and rc != scope.get("window_class"): + continue + ok = True + for k, v in props.items(): + if k == "control_type": + rv = r.get("control_type") or r.get("role") or "" + elif k == "role": + rv = r.get("role") or r.get("control_type") or "" + elif k == "text": + rv = r.get("text") or r.get("value") or r.get("name") or "" + elif k == "test_id": + rv = r.get("test_id") or r.get("automation_id") or "" + else: + rv = r.get(k) + if str(rv or "") != str(v): + ok = False + break + if ok: + n += 1 + return n diff --git a/tools/ui_inspector/locators/browser.py b/tools/ui_inspector/locators/browser.py new file mode 100644 index 0000000..69c2e42 --- /dev/null +++ b/tools/ui_inspector/locators/browser.py @@ -0,0 +1,190 @@ +"""Browser locator compiler (role / test id / label / placeholder / text / css id).""" + +from __future__ import annotations + +from typing import Any, Optional + +from models.locator import LocatorCandidate + +from .base import count_matches, rank_and_select_primary + + +def _expr(strategy: str, props: dict[str, Any]) -> str: + parts = [f"{k}={v!r}" for k, v in props.items() if v not in (None, "")] + return f"{strategy}:" + ",".join(parts) + + +def compile_browser_locators( + element: dict[str, Any], + *, + rows: Optional[list[dict[str, Any]]] = None, + frame: Optional[dict[str, Any]] = None, +) -> list[LocatorCandidate]: + """ + Generate ordered Playwright-oriented locator candidates. + + Priority roughly: role+name → test id → label → placeholder → text → css id → scoped. + Coordinates are never emitted as production candidates. + """ + el = element or {} + rows = list(rows or []) + scope = {"frame_url": (frame or {}).get("url") or "", "frame_name": (frame or {}).get("name") or ""} + cands: list[LocatorCandidate] = [] + + role = str(el.get("role") or el.get("control_type") or "").strip() + name = str(el.get("name") or "").strip() + text = str(el.get("text") or el.get("value") or "").strip() + is_editable = bool(el.get("contenteditable")) + if is_editable: + # Draft.js/contenteditable often exposes current user text as its + # accessible name. That value is payload, never element identity. + current_value = str(el.get("value") or "").strip() + if name == current_value or name == text: + name = "" + text = "" + test_id = str(el.get("test_id") or el.get("data_testid") or el.get("automation_id") or "").strip() + label = str(el.get("label") or "").strip() + placeholder = str(el.get("placeholder") or "").strip() + name_attr = str(el.get("name_attr") or "").strip() + href = str(el.get("href") or "").strip() + css_id = str(el.get("id") or el.get("css_id") or "").strip() + css_path = str(el.get("css") or el.get("css_path") or "").strip() + stable_css = str(el.get("stable_css") or "").strip() + semantic_role = str(el.get("semantic_role") or "").strip() + semantic_name = str(el.get("semantic_name") or "").strip() + # Prefer existing primary_locator from page_dump if present + existing = el.get("primary_locator") or {} + if isinstance(existing, dict) and existing.get("expr"): + props = dict(existing.get("properties") or {}) + if not props and existing.get("strategy"): + props = {"expr": existing.get("expr")} + c = LocatorCandidate( + strategy=str(existing.get("strategy") or "primary"), + scope=scope, + properties=props or {"expression": existing.get("expr")}, + expression=str(existing.get("expr") or ""), + stability_score=0.85, + reasons=["from_page_dump_primary_locator"], + ) + if rows and props: + c.mark_counts(count_matches(rows, properties=props, scope=scope)) + c.validation = "snapshot_only" + elif rows and existing.get("expr"): + # Cannot count without structured props — leave unique unknown + c.validation = "snapshot_only" + cands.append(c) + + for pw in el.get("playwright_candidates") or []: + if not isinstance(pw, dict): + continue + props = dict(pw.get("properties") or {}) + expr = str(pw.get("expr") or pw.get("expression") or "") + strat = str(pw.get("strategy") or "playwright") + c = LocatorCandidate( + strategy=strat, + scope=scope, + properties=props or {"expression": expr}, + expression=expr, + stability_score=float(pw.get("score") or 0.7), + uses_index=bool(pw.get("uses_index")), + uses_coordinates=bool(pw.get("uses_coordinates")), + reasons=["from_playwright_candidates"], + ) + if rows and props: + c.mark_counts(count_matches(rows, properties=props, scope=scope)) + c.validation = "snapshot_only" + cands.append(c) + + def add(strategy: str, props: dict[str, Any], score: float, reason: str) -> None: + if not props or not any(str(v).strip() for v in props.values()): + return + c = LocatorCandidate( + strategy=strategy, + scope=dict(scope), + properties=dict(props), + expression=_expr(strategy, props), + stability_score=score, + reasons=[reason], + ) + if rows: + c.mark_counts(count_matches(rows, properties=props, scope=scope)) + c.validation = "snapshot_only" + cands.append(c) + + if role and name: + add("role", {"role": role, "name": name}, 0.95, "role_name") + elif role and is_editable: + add("role", {"role": role}, 0.78, "editable_role") + if semantic_role and semantic_name: + add( + "semantic_role", + {"role": semantic_role, "name": semantic_name}, + 0.86, + "semantic_target_inferred_from_form_scope", + ) + if test_id: + add("test_id", {"test_id": test_id}, 0.93, "test_id") + if label: + add("label", {"label": label}, 0.88, "label") + if placeholder: + add("placeholder", {"placeholder": placeholder}, 0.84, "placeholder") + if name_attr: + add("name_attr", {"name_attr": name_attr}, 0.89, "html_name_attribute") + if href and role.lower() == "link": + add("href", {"href": href}, 0.91, "link_destination") + if text and len(text) <= 80: + add("text", {"text": text}, 0.75, "visible_text") + if css_id and not css_id[0].isdigit(): + add("css_id", {"id": css_id}, 0.8, "css_id") + if stable_css: + c = LocatorCandidate( + strategy="stable_css", + scope=dict(scope), + properties={"stable_css": stable_css}, + expression=f"css:{stable_css}", + stability_score=0.78 if ("#" in stable_css or "data-testid" in stable_css) else 0.62, + reasons=["stable_attribute_selector", "live_unique_required"], + ) + c.production_eligible = False + cands.append(c) + if css_path: + # Full/scoped CSS is a last-resort executable locator. It only becomes + # production eligible after live validation proves it is unique. + c = LocatorCandidate( + strategy="css", + scope=dict(scope), + properties={"css": css_path}, + expression=f"css:{css_path}", + stability_score=0.48 if "#" in css_path else 0.32, + reasons=["event_dom_path", "live_unique_required"], + ) + c.production_eligible = False + cands.append(c) + + # Never add coordinate locator as candidate for production + if el.get("bounds") or el.get("rect"): + c = LocatorCandidate( + strategy="bounding_rect", + scope=dict(scope), + properties={"bounds": el.get("bounds") or el.get("rect")}, + expression="coordinates:(evidence_only)", + stability_score=0.05, + uses_coordinates=True, + production_eligible=False, + reasons=["coordinates_evidence_only"], + ) + c.unique = False + c.ambiguous = True + cands.append(c) + + # Re-validate production_eligible after counts + for c in cands: + if c.actual_count is not None: + c.mark_counts(int(c.actual_count), expected=int(c.expected_count or 1)) + elif c.uses_coordinates: + c.production_eligible = False + + primary, _ = rank_and_select_primary(cands) + # Ensure ambiguous never becomes primary via side effect — rank handles it. + del primary + return cands diff --git a/tools/ui_inspector/locators/catalog.py b/tools/ui_inspector/locators/catalog.py new file mode 100644 index 0000000..c8f7731 --- /dev/null +++ b/tools/ui_inspector/locators/catalog.py @@ -0,0 +1,294 @@ +"""Element Catalog: merge observations, locator history, parameterization.""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any, Optional + +from models.element import ElementCatalog, ElementRef, element_fingerprint +from models.locator import LocatorCandidate +from models.serialize import to_dict, write_model_json +from redaction import redact_element_fields +from core.policies.element_identity import element_identity_key + +from .base import rank_and_select_primary +from .browser import compile_browser_locators +from .validator import validate_locator_against_rows +from .windows import compile_windows_locators + +_DYNAMIC_VALUE_HINTS = re.compile( + r"(联系人|搜索|search|contact|user|chat|title|name)", + re.I, +) + + +def suggest_logical_name(element: dict[str, Any]) -> str: + aid = str(element.get("automation_id") or "").strip() + name = str(element.get("name") or "").strip() + role = str(element.get("control_type") or element.get("role") or "el").strip() + text = str(element.get("text") or "").strip() + semantic_name = str(element.get("semantic_name") or "").strip() + if element.get("contenteditable"): + base = aid or str(element.get("placeholder") or element.get("label") or "") or role + else: + base = aid or semantic_name or name or (text if 0 < len(text) <= 48 else "") + if not base: + for ancestor in element.get("parent_chain") or []: + ancestor_id = str((ancestor or {}).get("id") or "").strip() + ancestor_text = str((ancestor or {}).get("text") or "").strip() + if ancestor_id: + base = f"{ancestor_id}_{role}" + break + if ancestor_text and len(ancestor_text) <= 32: + base = f"{ancestor_text}_{role}" + break + base = base or role + base = re.sub(r"[^\w\u4e00-\u9fff]+", "_", base, flags=re.UNICODE).strip("_") + base = base[:48] or "element" + return base.lower() if base.isascii() else base + + +def _maybe_parameterize(value: str, *, field: str) -> tuple[str, dict[str, str]]: + """Replace likely dynamic business values with ${param} placeholders.""" + v = (value or "").strip() + if not v or len(v) > 64: + return v, {} + if field in {"value", "text", "name"} and _DYNAMIC_VALUE_HINTS.search(field): + return "${value}", {"value": v} + # Contact-like short Chinese/ASCII names in value when name is generic + if field == "value" and 1 < len(v) <= 32 and not v.startswith("${"): + # Only parameterize when looks like a person/search term (letters/CJK, not chrome) + if re.fullmatch(r"[\w\u4e00-\u9fff \-_.@]+", v, flags=re.UNICODE): + chrome = {"关闭", "最小化", "最大化", "取消", "确定", "发送", "搜索"} + if v not in chrome: + return "${contact_name}", {"contact_name": v} + return v, {} + + +class ElementCatalogService: + def __init__(self, flow_dir: Path, *, flow_id: str = "") -> None: + self.flow_dir = Path(flow_dir) + self.elements_dir = self.flow_dir / "elements" + self.elements_dir.mkdir(parents=True, exist_ok=True) + self.catalog_path = self.flow_dir / "element_catalog.json" + self.catalog = ElementCatalog(flow_id=flow_id) + self._by_fp: dict[str, ElementRef] = {} + self._by_identity: dict[str, ElementRef] = {} + self._by_name: dict[str, ElementRef] = {} + + def load(self) -> None: + if not self.catalog_path.exists(): + return + import json + from models.serialize import from_dict + + data = json.loads(self.catalog_path.read_text(encoding="utf-8")) + self.catalog.flow_id = data.get("flow_id") or self.catalog.flow_id + for item in data.get("elements") or []: + # Reconstruct LocatorCandidate lists lightly + locs = [] + for lc in item.get("locator_candidates") or []: + locs.append(from_dict(LocatorCandidate, lc)) + primary = None + if item.get("primary_locator"): + primary = from_dict(LocatorCandidate, item["primary_locator"]) + fallbacks = [ + from_dict(LocatorCandidate, x) for x in (item.get("fallback_locators") or []) + ] + el = ElementRef( + id=item.get("id") or "", + logical_name=item.get("logical_name") or "", + surface=item.get("surface") or "windows", + backend=item.get("backend") or "", + app=item.get("app") or {}, + window_scope=item.get("window_scope") or {}, + element_scope=item.get("element_scope") or {}, + role=item.get("role") or "", + control_type=item.get("control_type") or "", + name=item.get("name") or "", + value=item.get("value") or "", + automation_id=item.get("automation_id") or "", + class_name=item.get("class_name") or "", + framework_id=item.get("framework_id") or "", + states=item.get("states") or {}, + bounds=item.get("bounds") or {}, + actions=list(item.get("actions") or []), + parent_chain=list(item.get("parent_chain") or []), + raw=item.get("raw") or {}, + fingerprint=item.get("fingerprint") or "", + sensitive=bool(item.get("sensitive")), + locator_candidates=locs, + primary_locator=primary, + fallback_locators=fallbacks, + expected_count=int(item.get("expected_count") or 1), + verified_count=item.get("verified_count"), + verification_level=item.get("verification_level") or "none", + stability_score=float(item.get("stability_score") or 0.0), + verified_at=item.get("verified_at"), + first_seen=item.get("first_seen") or "", + last_seen=item.get("last_seen") or "", + observation_count=int(item.get("observation_count") or 1), + locator_history=list(item.get("locator_history") or []), + parameterized=dict(item.get("parameterized") or {}), + ) + if el.fingerprint: + self._by_fp[el.fingerprint] = el + identity = str((el.raw or {}).get("_identity_key") or "") + if not identity: + identity = element_identity_key( + el.raw or to_dict(el), + surface=el.surface, + page_url=str((el.element_scope or {}).get("page_url") or ""), + frame_name=str((el.element_scope or {}).get("frame_name") or ""), + ) + self._by_identity[identity] = el + if el.logical_name: + self._by_name[el.logical_name] = el + self.catalog.elements.append(el) + + def save(self) -> None: + from datetime import datetime + + self.catalog.updated_at = datetime.now().astimezone().isoformat() + write_model_json(self.catalog_path, self.catalog) + for el in self.catalog.elements: + write_model_json(self.elements_dir / f"{el.id}.json", el) + + def upsert_from_snapshot( + self, + element: dict[str, Any], + *, + surface: str, + backend: str = "", + window_scope: Optional[dict[str, Any]] = None, + rows: Optional[list[dict[str, Any]]] = None, + logical_name: Optional[str] = None, + frame: Optional[dict[str, Any]] = None, + live_counts: Optional[dict[str, int]] = None, + ) -> ElementRef: + cleaned, sensitive = redact_element_fields(dict(element or {})) + win = dict(window_scope or {}) + value = str(cleaned.get("value") or "") + is_editable = bool(cleaned.get("contenteditable")) + param_value, params = _maybe_parameterize(value, field="value") + if params: + cleaned["value"] = param_value + + fp = element_fingerprint( + surface=surface, + role=str(cleaned.get("control_type") or cleaned.get("role") or ""), + name=str(cleaned.get("name") or ""), + automation_id=str(cleaned.get("automation_id") or ""), + class_name=str(cleaned.get("class_name") or ""), + value=str(cleaned.get("value") or ""), + window_scope=win, + dom_fingerprint=str(cleaned.get("fingerprint") or cleaned.get("css") or ""), + page_scope=str((frame or {}).get("url") or win.get("url") or ""), + contenteditable=is_editable, + ) + + identity = element_identity_key( + cleaned, + surface=surface, + page_url=str((frame or {}).get("url") or win.get("url") or ""), + frame_name=str((frame or {}).get("name") or ""), + ) + cleaned["_identity_key"] = identity + + existing = self._by_identity.get(identity) or self._by_fp.get(fp) + if existing is None and logical_name and logical_name in self._by_name: + existing = self._by_name[logical_name] + + if surface == "browser": + cands = compile_browser_locators(cleaned, rows=rows, frame=frame) + else: + cands = compile_windows_locators(cleaned, rows=rows, window_scope=win) + + for c in cands: + if rows is not None and not c.uses_coordinates: + validate_locator_against_rows(c, rows, level="snapshot_only") + if live_counts is not None and c.strategy in live_counts: + c.mark_counts(int(live_counts[c.strategy])) + c.validation = "live" + + primary, fallbacks = rank_and_select_primary(cands) + # Ambiguous / coordinates never primary + if primary is not None and (primary.ambiguous or not primary.production_eligible): + primary = None + + lname = logical_name or (existing.logical_name if existing else "") or suggest_logical_name( + cleaned + ) + + if existing: + # Merge observation; keep locator history + if primary is not None: + old = existing.primary_locator + if old is not None and to_dict(old) != to_dict(primary): + existing.locator_history.append( + {"replaced_at": existing.last_seen, "locator": to_dict(old)} + ) + existing.primary_locator = primary + existing.locator_candidates = cands + existing.fallback_locators = fallbacks + existing.touch() + existing.raw = cleaned + existing.bounds = dict(cleaned.get("bounds") or cleaned.get("rect") or existing.bounds or {}) + if params: + existing.parameterized.update(params) + existing.sensitive = existing.sensitive or sensitive + existing.verification_level = "live" if live_counts else "snapshot_only" + self._by_fp[fp] = existing + self._by_identity[identity] = existing + if primary and primary.actual_count is not None: + existing.verified_count = primary.actual_count + self.save() + return existing + + el = ElementRef( + logical_name=lname, + surface=surface, + backend=backend, + window_scope=win, + element_scope={ + "page_url": str((frame or {}).get("url") or win.get("url") or ""), + "frame_name": str((frame or {}).get("name") or ""), + }, + role=str(cleaned.get("role") or ""), + control_type=str(cleaned.get("control_type") or ""), + name=str(cleaned.get("name") or ""), + value=str(cleaned.get("value") or ""), + automation_id=str(cleaned.get("automation_id") or ""), + class_name=str(cleaned.get("class_name") or ""), + framework_id=str(cleaned.get("framework_id") or ""), + states={ + "is_enabled": cleaned.get("is_enabled"), + "is_offscreen": cleaned.get("is_offscreen"), + }, + bounds=dict(cleaned.get("bounds") or cleaned.get("rect") or {}), + parent_chain=list(cleaned.get("parent_chain") or []), + raw=cleaned, + fingerprint=fp, + sensitive=sensitive, + locator_candidates=cands, + primary_locator=primary, + fallback_locators=fallbacks, + verification_level="live" if live_counts else "snapshot_only", + verified_count=(primary.actual_count if primary else None), + parameterized=params, + stability_score=float(primary.stability_score) if primary else 0.0, + ) + # Ensure unique logical_name + base = el.logical_name + i = 2 + while el.logical_name in self._by_name: + el.logical_name = f"{base}_{i}" + i += 1 + + self.catalog.elements.append(el) + self._by_fp[fp] = el + self._by_identity[identity] = el + self._by_name[el.logical_name] = el + self.save() + return el diff --git a/tools/ui_inspector/locators/validator.py b/tools/ui_inspector/locators/validator.py new file mode 100644 index 0000000..a351e98 --- /dev/null +++ b/tools/ui_inspector/locators/validator.py @@ -0,0 +1,50 @@ +"""Locator uniqueness validation against snapshot / live tree rows.""" + +from __future__ import annotations + +from typing import Any, Optional + +from models.locator import LocatorCandidate + +from .base import count_matches + + +def validate_locator_against_rows( + candidate: LocatorCandidate, + rows: list[dict[str, Any]], + *, + level: str = "snapshot_only", + expected_count: int = 1, +) -> LocatorCandidate: + """ + Validate structured locator properties against a row list. + + Levels supported in this phase: snapshot_only | live_tree + """ + actual = count_matches( + rows, + properties=candidate.properties, + scope=candidate.scope, + ) + candidate.mark_counts(actual, expected=expected_count) + candidate.validation = level + if not candidate.unique: + candidate.reasons = list(candidate.reasons or []) + if "ambiguous_match_count" not in candidate.reasons: + candidate.reasons.append("ambiguous_match_count") + candidate.production_eligible = False + if candidate.uses_coordinates: + candidate.production_eligible = False + if "coordinates_never_primary" not in (candidate.reasons or []): + candidate.reasons = list(candidate.reasons or []) + ["coordinates_never_primary"] + return candidate + + +def choose_production_primary( + candidates: list[LocatorCandidate], +) -> Optional[LocatorCandidate]: + """Return first production-eligible unique non-coordinate locator, else None.""" + for c in candidates: + if c.production_eligible and c.unique and not c.uses_coordinates: + return c + return None diff --git a/tools/ui_inspector/locators/windows.py b/tools/ui_inspector/locators/windows.py new file mode 100644 index 0000000..cc591c4 --- /dev/null +++ b/tools/ui_inspector/locators/windows.py @@ -0,0 +1,180 @@ +"""Windows UIA locator compiler (Name ≠ Value; no coordinate primary).""" + +from __future__ import annotations + +from typing import Any, Optional + +from models.locator import LocatorCandidate + +from .base import count_matches, rank_and_select_primary + + +def _expr(strategy: str, props: dict[str, Any]) -> str: + parts = [f"{k}={v!r}" for k, v in props.items() if v not in (None, "")] + return f"{strategy}:" + ",".join(parts) + + +def compile_windows_locators( + element: dict[str, Any], + *, + rows: Optional[list[dict[str, Any]]] = None, + window_scope: Optional[dict[str, Any]] = None, +) -> list[LocatorCandidate]: + """ + Priority: + 1. Window scope + AutomationId + ControlType + 2. Window scope + stable Name + ControlType + 3. Window scope + Name + Value + ControlType + 4. Business parent + child attrs + 5. Visible Value + ControlType + 6. Relative ancestor scope + 7. Index with business reason (low score) + 8. BoundingRectangle evidence only + """ + el = element or {} + rows = list(rows or []) + win = dict(window_scope or el.get("window_scope") or {}) + scope = { + "window_class": win.get("class_name") or win.get("window_class") or "", + "window_title": win.get("title") or win.get("window_title") or "", + "process_name": win.get("process_name") or "", + } + cands: list[LocatorCandidate] = [] + + aid = str(el.get("automation_id") or "").strip() + name = str(el.get("name") or "").strip() + value = str(el.get("value") or "").strip() + ctype = str(el.get("control_type") or el.get("role") or "").strip() + class_name = str(el.get("class_name") or "").strip() + parent_chain = list(el.get("parent_chain") or []) + + def add( + strategy: str, + props: dict[str, Any], + score: float, + reason: str, + *, + uses_index: bool = False, + uses_coordinates: bool = False, + ) -> None: + if not props or not any(str(v).strip() for v in props.values() if not isinstance(v, (dict, list))): + return + c = LocatorCandidate( + strategy=strategy, + scope=dict(scope), + properties=dict(props), + expression=_expr(strategy, props), + stability_score=score, + uses_index=uses_index, + uses_coordinates=uses_coordinates, + reasons=[reason], + ) + if rows: + c.mark_counts(count_matches(rows, properties=props, scope=scope)) + c.validation = "snapshot_only" + elif uses_coordinates: + c.production_eligible = False + c.unique = False + c.ambiguous = True + cands.append(c) + + if aid and ctype: + add( + "window_automation_id_type", + {"automation_id": aid, "control_type": ctype}, + 0.96, + "automation_id_control_type", + ) + elif aid: + add("window_automation_id", {"automation_id": aid}, 0.9, "automation_id") + + if name and ctype and name.lower() not in {"item", "pane", "unknown"}: + add( + "window_name_type", + {"name": name, "control_type": ctype}, + 0.88, + "stable_name_control_type", + ) + + if name and value and ctype: + add( + "window_name_value_type", + {"name": name, "value": value, "control_type": ctype}, + 0.86, + "name_value_control_type", + ) + + # Business parent + child + if parent_chain: + parent = parent_chain[0] if isinstance(parent_chain[0], dict) else {} + parent_name = str(parent.get("name") or parent.get("automation_id") or "").strip() + if parent_name and (aid or name or value): + props: dict[str, Any] = {"parent_name": parent_name} + if aid: + props["automation_id"] = aid + if name: + props["name"] = name + if value: + props["value"] = value + if ctype: + props["control_type"] = ctype + add("parent_scoped", props, 0.82, "business_parent_scope") + + if value and ctype and len(value) <= 64: + add( + "value_type", + {"value": value, "control_type": ctype}, + 0.78, + "visible_value_control_type", + ) + + if class_name and ctype: + add( + "class_type", + {"class_name": class_name, "control_type": ctype}, + 0.55, + "class_name_control_type", + ) + + # Index only with explicit business reason — low score, not auto primary + idx = el.get("index") + if idx is not None and (aid or name): + props = {"index": int(idx)} + if aid: + props["automation_id"] = aid + if name: + props["name"] = name + if ctype: + props["control_type"] = ctype + add( + "indexed", + props, + 0.35, + "index_requires_business_justification", + uses_index=True, + ) + + bounds = el.get("bounds") or el.get("rect") or el.get("bounding_rectangle") + if bounds: + c = LocatorCandidate( + strategy="bounding_rectangle", + scope=dict(scope), + properties={"bounds": bounds}, + expression="coordinates:(evidence_only)", + stability_score=0.05, + uses_coordinates=True, + production_eligible=False, + unique=False, + ambiguous=True, + reasons=["coordinates_never_primary"], + ) + cands.append(c) + + for c in cands: + if c.actual_count is not None: + c.mark_counts(int(c.actual_count), expected=int(c.expected_count or 1)) + if c.uses_coordinates: + c.production_eligible = False + + rank_and_select_primary(cands) + return cands diff --git a/tools/ui_inspector/models/__init__.py b/tools/ui_inspector/models/__init__.py new file mode 100644 index 0000000..2aa20b5 --- /dev/null +++ b/tools/ui_inspector/models/__init__.py @@ -0,0 +1,22 @@ +"""UI Inspector v2 data models (Action / State / Element / Flow).""" + +from .action import ACTION_TYPES, ActionFlow, ActionRecord +from .element import ElementRef +from .locator import LocatorCandidate +from .observation import UIObservation +from .serialize import from_dict, to_dict, write_model_json +from .state import UIState, compute_state_fingerprint + +__all__ = [ + "ACTION_TYPES", + "ActionFlow", + "ActionRecord", + "ElementRef", + "LocatorCandidate", + "UIState", + "UIObservation", + "compute_state_fingerprint", + "from_dict", + "to_dict", + "write_model_json", +] diff --git a/tools/ui_inspector/models/action.py b/tools/ui_inspector/models/action.py new file mode 100644 index 0000000..9a88597 --- /dev/null +++ b/tools/ui_inspector/models/action.py @@ -0,0 +1,119 @@ +"""ActionRecord + ActionFlow models.""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Optional + +SCHEMA_ACTION = "openclaw.ui_inspector.action.v1" +SCHEMA_ACTION_FLOW = "openclaw.ui_inspector.action_flow.v1" + +# Full vocabulary — capability flags live elsewhere for unimplemented types. +ACTION_TYPES: tuple[str, ...] = ( + "click", + "double_click", + "right_click", + "fill", + "type", + "clear", + "press_key", + "hotkey", + "select", + "toggle", + "hover", + "scroll", + "drag_drop", + "focus", + "window_switch", + "navigation", + "checkpoint", + "wait", + "verify", + "unknown", +) + +# Capability status for this Inspector release. +ACTION_CAPABILITY: dict[str, str] = { + "click": "implemented", + "double_click": "schema_only", + "right_click": "schema_only", + "fill": "implemented", + "type": "implemented", # aggregated into fill/type + "clear": "schema_only", + "press_key": "implemented", + "hotkey": "schema_only", + "select": "schema_only", + "toggle": "schema_only", + "hover": "schema_only", + "scroll": "schema_only", + "drag_drop": "schema_only", + "focus": "schema_only", + "window_switch": "schema_only", + "navigation": "partial", + "checkpoint": "implemented", + "wait": "schema_only", + "verify": "schema_only", + "unknown": "fallback", +} + + +def _now_iso() -> str: + return datetime.now().astimezone().isoformat() + + +def _new_id(prefix: str = "act") -> str: + return f"{prefix}_{uuid.uuid4().hex[:12]}" + + +@dataclass +class ActionRecord: + schema: str = SCHEMA_ACTION + id: str = field(default_factory=lambda: _new_id("act")) + sequence: int = 0 + action_type: str = "unknown" + started_at: str = field(default_factory=_now_iso) + ended_at: Optional[str] = None + surface: str = "windows" + target_element_id: Optional[str] = None + target_snapshot: dict[str, Any] = field(default_factory=dict) + input: dict[str, Any] = field(default_factory=dict) + sensitive: bool = False + before_state_id: Optional[str] = None + after_state_id: Optional[str] = None + before_observation_id: Optional[str] = None + after_observation_id: Optional[str] = None + observed_changes: list[dict[str, Any]] = field(default_factory=list) + proposed_postconditions: list[dict[str, Any]] = field(default_factory=list) + status: str = "ok" # ok | partial | failed | skipped + confidence: float = 0.0 + evidence: dict[str, Any] = field(default_factory=dict) + raw_event: dict[str, Any] = field(default_factory=dict) + raw_event_ids: list[str] = field(default_factory=list) + capability: str = "" + + def __post_init__(self) -> None: + if self.action_type not in ACTION_TYPES: + self.action_type = "unknown" + if not self.capability: + self.capability = ACTION_CAPABILITY.get(self.action_type, "schema_only") + + +@dataclass +class ActionFlow: + schema: str = SCHEMA_ACTION_FLOW + flow_id: str = "" + flow_name: str = "" + created_at: str = field(default_factory=_now_iso) + ended_at: Optional[str] = None + states: list[str] = field(default_factory=list) # state ids in order seen + observations: list[str] = field(default_factory=list) # every concrete capture + actions: list[str] = field(default_factory=list) # action ids in sequence + elements: list[str] = field(default_factory=list) # element ids + entry_state_id: Optional[str] = None + final_state_id: Optional[str] = None + metadata: dict[str, Any] = field(default_factory=dict) + recording_mode: str = "auto" # auto | guided | checkpoint | legacy_interval + surface: str = "" + status: str = "open" diff --git a/tools/ui_inspector/models/element.py b/tools/ui_inspector/models/element.py new file mode 100644 index 0000000..757e562 --- /dev/null +++ b/tools/ui_inspector/models/element.py @@ -0,0 +1,126 @@ +"""ElementRef model for Element Catalog.""" + +from __future__ import annotations + +import hashlib +import json +import uuid +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Optional + +from .locator import LocatorCandidate + +SCHEMA_ELEMENT = "openclaw.ui_inspector.element.v1" +SCHEMA_CATALOG = "openclaw.ui_inspector.element_catalog.v1" + + +def _now_iso() -> str: + return datetime.now().astimezone().isoformat() + + +def _new_id(prefix: str = "el") -> str: + return f"{prefix}_{uuid.uuid4().hex[:12]}" + + +def element_fingerprint( + *, + surface: str, + role: str = "", + name: str = "", + automation_id: str = "", + class_name: str = "", + value: str = "", + window_scope: Optional[dict[str, Any]] = None, + dom_fingerprint: str = "", + page_scope: str = "", + contenteditable: bool = False, +) -> str: + payload = { + "surface": surface or "", + "role": role or "", + "name": "" if contenteditable else (name or ""), + "automation_id": automation_id or "", + "class_name": class_name or "", + # Value is identity only when short/stable-looking (not long free text). + "value": "" if contenteditable else ((value or "") if len(value or "") <= 64 else ""), + "dom_fingerprint": dom_fingerprint or "", + "page_scope": page_scope or "", + "window_scope": { + "class_name": (window_scope or {}).get("class_name") or "", + "title": (window_scope or {}).get("title") or "", + }, + } + raw = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + + +@dataclass +class ElementRef: + schema: str = SCHEMA_ELEMENT + id: str = field(default_factory=lambda: _new_id("el")) + logical_name: str = "" + surface: str = "windows" + backend: str = "" + app: dict[str, Any] = field(default_factory=dict) + window_scope: dict[str, Any] = field(default_factory=dict) + element_scope: dict[str, Any] = field(default_factory=dict) + role: str = "" + control_type: str = "" + name: str = "" + value: str = "" + automation_id: str = "" + class_name: str = "" + framework_id: str = "" + states: dict[str, Any] = field(default_factory=dict) + bounds: dict[str, Any] = field(default_factory=dict) + actions: list[str] = field(default_factory=list) + parent_chain: list[dict[str, Any]] = field(default_factory=list) + raw: dict[str, Any] = field(default_factory=dict) + fingerprint: str = "" + sensitive: bool = False + locator_candidates: list[LocatorCandidate] = field(default_factory=list) + primary_locator: Optional[LocatorCandidate] = None + fallback_locators: list[LocatorCandidate] = field(default_factory=list) + expected_count: int = 1 + verified_count: Optional[int] = None + verification_level: str = "none" + stability_score: float = 0.0 + verified_at: Optional[str] = None + first_seen: str = field(default_factory=_now_iso) + last_seen: str = field(default_factory=_now_iso) + observation_count: int = 1 + locator_history: list[dict[str, Any]] = field(default_factory=list) + parameterized: dict[str, str] = field(default_factory=dict) + + def ensure_fingerprint(self) -> str: + if self.fingerprint: + return self.fingerprint + self.fingerprint = element_fingerprint( + surface=self.surface, + role=self.role or self.control_type, + name=self.name, + automation_id=self.automation_id, + class_name=self.class_name, + value=self.value, + window_scope=self.window_scope, + dom_fingerprint=str(self.raw.get("fingerprint") or self.raw.get("css") or ""), + page_scope=str(self.element_scope.get("page_url") or ""), + contenteditable=bool(self.raw.get("contenteditable")), + ) + return self.fingerprint + + def touch(self) -> None: + self.last_seen = _now_iso() + self.observation_count = int(self.observation_count or 0) + 1 + + +@dataclass +class ElementCatalog: + schema: str = SCHEMA_CATALOG + flow_id: str = "" + updated_at: str = field(default_factory=_now_iso) + elements: list[ElementRef] = field(default_factory=list) + + def by_logical_name(self) -> dict[str, ElementRef]: + return {e.logical_name: e for e in self.elements if e.logical_name} diff --git a/tools/ui_inspector/models/flow.py b/tools/ui_inspector/models/flow.py new file mode 100644 index 0000000..8883ca7 --- /dev/null +++ b/tools/ui_inspector/models/flow.py @@ -0,0 +1,11 @@ +"""Flow-level helpers (re-export ActionFlow for models.flow import path).""" + +from .action import ActionFlow, ActionRecord +from .element import ElementCatalog, ElementRef +from .state import UIState +from .observation import UIObservation + +__all__ = [ + "ActionFlow", "ActionRecord", "ElementCatalog", "ElementRef", "UIState", + "UIObservation", +] diff --git a/tools/ui_inspector/models/locator.py b/tools/ui_inspector/models/locator.py new file mode 100644 index 0000000..d0c5864 --- /dev/null +++ b/tools/ui_inspector/models/locator.py @@ -0,0 +1,40 @@ +"""Structured locator candidate model.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Optional + + +SCHEMA_LOCATOR = "openclaw.ui_inspector.locator_candidate.v1" + + +@dataclass +class LocatorCandidate: + schema: str = SCHEMA_LOCATOR + strategy: str = "" + scope: dict[str, Any] = field(default_factory=dict) + properties: dict[str, Any] = field(default_factory=dict) + expression: str = "" + expected_count: int = 1 + actual_count: Optional[int] = None + unique: Optional[bool] = None + stability_score: float = 0.0 + uses_index: bool = False + uses_coordinates: bool = False + validation: str = "none" # snapshot_only | live_tree | cross_state | cross_session | none + reasons: list[str] = field(default_factory=list) + ambiguous: bool = False + production_eligible: bool = False + + def mark_counts(self, actual: int, *, expected: int = 1) -> None: + self.expected_count = int(expected) + self.actual_count = int(actual) + self.unique = self.actual_count == self.expected_count + self.ambiguous = not bool(self.unique) + # Coordinates / index never become production primary automatically. + self.production_eligible = bool( + self.unique + and not self.uses_coordinates + and not (self.uses_index and self.stability_score < 0.7) + ) diff --git a/tools/ui_inspector/models/observation.py b/tools/ui_inspector/models/observation.py new file mode 100644 index 0000000..8db5713 --- /dev/null +++ b/tools/ui_inspector/models/observation.py @@ -0,0 +1,47 @@ +"""Per-capture evidence model. + +UIState represents a deduplicated semantic state. UIObservation represents +one concrete capture and is therefore never deduplicated. +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Optional + +SCHEMA_OBSERVATION = "openclaw.ui_inspector.observation.v1" + + +def _now_iso() -> str: + return datetime.now().astimezone().isoformat() + + +def _new_id() -> str: + return f"obs_{uuid.uuid4().hex[:12]}" + + +@dataclass +class UIObservation: + schema: str = SCHEMA_OBSERVATION + id: str = field(default_factory=_new_id) + captured_at: str = field(default_factory=_now_iso) + state_id: str = "" + previous_observation_id: Optional[str] = None + change_reason: str = "capture" + surface: str = "windows" + driver: str = "" + page_id: str = "" + document_id: str = "" + frame_id: str = "" + url: str = "" + page_title: str = "" + tree_path: str = "" + index_path: str = "" + locatable_path: str = "" + screenshot_path: str = "" + stats: dict[str, Any] = field(default_factory=dict) + selection: dict[str, Any] = field(default_factory=dict) + may_contain_sensitive_data: bool = False + diff --git a/tools/ui_inspector/models/serialize.py b/tools/ui_inspector/models/serialize.py new file mode 100644 index 0000000..f566a8e --- /dev/null +++ b/tools/ui_inspector/models/serialize.py @@ -0,0 +1,64 @@ +"""Stable JSON serialization helpers for v2 models.""" + +from __future__ import annotations + +import json +from dataclasses import asdict, is_dataclass +from datetime import datetime +from pathlib import Path +from typing import Any, TypeVar + +T = TypeVar("T") + + +def _normalize(obj: Any) -> Any: + if obj is None or isinstance(obj, (str, int, float, bool)): + return obj + if isinstance(obj, datetime): + return obj.isoformat() + if isinstance(obj, Path): + return str(obj).replace("\\", "/") + if is_dataclass(obj) and not isinstance(obj, type): + return {k: _normalize(v) for k, v in asdict(obj).items()} + if isinstance(obj, dict): + return {str(k): _normalize(v) for k, v in obj.items()} + if isinstance(obj, (list, tuple)): + return [_normalize(x) for x in obj] + return str(obj) + + +def to_dict(obj: Any) -> dict[str, Any]: + data = _normalize(obj) + if not isinstance(data, dict): + raise TypeError(f"expected dataclass/dict, got {type(obj)!r}") + return data + + +def from_dict(cls: type[T], data: dict[str, Any]) -> T: + """Best-effort dataclass reconstruction (ignores unknown keys).""" + if not is_dataclass(cls): + raise TypeError(f"{cls!r} is not a dataclass") + import dataclasses + + fields = {f.name for f in dataclasses.fields(cls)} + kwargs = {k: v for k, v in (data or {}).items() if k in fields} + return cls(**kwargs) # type: ignore[call-arg] + + +def dumps_json(obj: Any, *, indent: int = 2) -> str: + return json.dumps( + to_dict(obj) if is_dataclass(obj) and not isinstance(obj, type) else _normalize(obj), + ensure_ascii=False, + indent=indent, + sort_keys=False, + ) + "\n" + + +def write_model_json(path: Path, obj: Any) -> None: + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(dumps_json(obj), encoding="utf-8") + + +def read_json(path: Path) -> Any: + return json.loads(Path(path).read_text(encoding="utf-8")) diff --git a/tools/ui_inspector/models/state.py b/tools/ui_inspector/models/state.py new file mode 100644 index 0000000..c483fe0 --- /dev/null +++ b/tools/ui_inspector/models/state.py @@ -0,0 +1,175 @@ +"""UIState model + stable business fingerprint.""" + +from __future__ import annotations + +import hashlib +import json +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Optional + +SCHEMA_STATE = "openclaw.ui_inspector.state.v1" + +# Keys ignored when hashing business identity. +_FINGERPRINT_EXCLUDE_TOP = { + "captured_at", + "id", + "previous_state_id", + "tree_path", + "index_path", + "locatable_path", + "screenshot_path", + "duplicate_of", + "observation_count", + "change_reason", +} + +_SESSION_KEYS = { + "pid", + "hwnd", + "process_id", + "thread_id", + "elapsed_ms", + "dump_ms", + "warmup_ms", + "timings", + "screenshot", + "screen_path", +} + + +def _now_iso() -> str: + return datetime.now().astimezone().isoformat() + + +def _new_id(prefix: str = "st") -> str: + return f"{prefix}_{uuid.uuid4().hex[:12]}" + + +def _strip_session(obj: Any) -> Any: + if isinstance(obj, dict): + out: dict[str, Any] = {} + for k, v in obj.items(): + lk = str(k).lower() + if lk in _SESSION_KEYS or lk.endswith("_ms") or lk.endswith("_path"): + continue + if lk in {"left", "top", "right", "bottom", "x", "y", "width", "height"}: + # Absolute coordinates are not primary identity. + continue + if lk == "rect" or lk == "bounds" or lk == "bounding_rectangle": + continue + out[str(k)] = _strip_session(v) + return out + if isinstance(obj, list): + return [_strip_session(x) for x in obj] + return obj + + +def _locatable_signature(rows: list[dict[str, Any]], *, limit: int = 120) -> list[dict[str, Any]]: + """Compact visible-business signature for fingerprinting.""" + sig: list[dict[str, Any]] = [] + for r in rows[:limit]: + if not isinstance(r, dict): + continue + entry = { + "control_type": r.get("control_type") or r.get("role") or "", + "name": r.get("name") or "", + "value": r.get("value") or r.get("text") or "", + "automation_id": r.get("automation_id") or r.get("test_id") or "", + "class_name": r.get("class_name") or "", + } + # Drop empty noise rows + if not any(str(v).strip() for v in entry.values()): + continue + sig.append(entry) + # Stable order + sig.sort( + key=lambda e: ( + str(e.get("control_type") or ""), + str(e.get("automation_id") or ""), + str(e.get("name") or ""), + str(e.get("value") or ""), + ) + ) + return sig + + +def compute_state_fingerprint( + *, + surface: str, + driver: str = "", + url: str = "", + page_title: str = "", + window: Optional[dict[str, Any]] = None, + locatable: Optional[list[dict[str, Any]]] = None, + app: Optional[dict[str, Any]] = None, + extra_business: Optional[dict[str, Any]] = None, +) -> str: + """ + SHA-256 of stable business identity. + + Excludes timestamps, pid/hwnd, screenshot paths, absolute coords, timings. + """ + win = dict(window or {}) + app_d = dict(app or {}) + payload = { + "surface": surface or "", + "driver": driver or "", + "url": url or win.get("url") or app_d.get("url") or "", + "page_title": page_title or "", + "window": { + "title": win.get("title") or app_d.get("title") or "", + "class_name": win.get("class_name") or app_d.get("class_name") or "", + "process_name": win.get("process_name") or app_d.get("process_name") or "", + }, + "locatable": _locatable_signature(list(locatable or [])), + "extra": _strip_session(extra_business or {}), + } + raw = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + + +@dataclass +class UIState: + schema: str = SCHEMA_STATE + id: str = field(default_factory=lambda: _new_id("st")) + surface: str = "windows" + driver: str = "" + captured_at: str = field(default_factory=_now_iso) + fingerprint: str = "" + previous_state_id: Optional[str] = None + change_reason: str = "" + app: dict[str, Any] = field(default_factory=dict) + window: dict[str, Any] = field(default_factory=dict) + url: str = "" + page_title: str = "" + tree_path: str = "" + index_path: str = "" + locatable_path: str = "" + screenshot_path: str = "" + stats: dict[str, Any] = field(default_factory=dict) + selection: dict[str, Any] = field(default_factory=dict) + duplicate_of: Optional[str] = None + observation_count: int = 1 + may_contain_sensitive_data: bool = False + # In-memory locatable used for fingerprint/diff (not always persisted as field). + locatable: list[dict[str, Any]] = field(default_factory=list, repr=False) + + def ensure_fingerprint(self) -> str: + if self.fingerprint: + return self.fingerprint + self.fingerprint = compute_state_fingerprint( + surface=self.surface, + driver=self.driver, + url=self.url, + page_title=self.page_title, + window=self.window, + locatable=self.locatable, + app=self.app, + ) + return self.fingerprint + + def bump_observation(self) -> None: + self.observation_count = int(self.observation_count or 0) + 1 + self.captured_at = _now_iso() diff --git a/tools/ui_inspector/recorder/__init__.py b/tools/ui_inspector/recorder/__init__.py new file mode 100644 index 0000000..43b0d27 --- /dev/null +++ b/tools/ui_inspector/recorder/__init__.py @@ -0,0 +1,13 @@ +"""Recorder package: observe user actions outside target processes.""" + +from .base import RawUserEvent, RecorderBase, RecorderState +from .input_aggregator import InputAggregator +from .manager import ActionRecordingManager + +__all__ = [ + "RawUserEvent", + "RecorderBase", + "RecorderState", + "InputAggregator", + "ActionRecordingManager", +] diff --git a/tools/ui_inspector/recorder/base.py b/tools/ui_inspector/recorder/base.py new file mode 100644 index 0000000..af1300b --- /dev/null +++ b/tools/ui_inspector/recorder/base.py @@ -0,0 +1,79 @@ +"""Recorder abstractions.""" + +from __future__ import annotations + +import threading +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Callable, Optional + + +def _now_iso() -> str: + return datetime.now().astimezone().isoformat() + + +@dataclass +class RawUserEvent: + kind: str # mouse_down | mouse_up | key | input | navigation | focus | checkpoint + surface: str + timestamp: str = field(default_factory=_now_iso) + button: str = "" + key: str = "" + text: str = "" + x: Optional[int] = None + y: Optional[int] = None + target: dict[str, Any] = field(default_factory=dict) + frame: dict[str, Any] = field(default_factory=dict) + url: str = "" + meta: dict[str, Any] = field(default_factory=dict) + from_inspector: bool = False + + +class RecorderState: + IDLE = "idle" + RECORDING = "recording" + PAUSED = "paused" + STOPPED = "stopped" + + +class RecorderBase(ABC): + """Process-external observer. No injection / hooks into target.""" + + def __init__( + self, + *, + on_event: Optional[Callable[[RawUserEvent], None]] = None, + exclude_hwnds: Optional[Callable[[], set[int]]] = None, + ) -> None: + self.on_event = on_event + self.exclude_hwnds = exclude_hwnds or (lambda: set()) + self.status = RecorderState.IDLE + self._thread: Optional[threading.Thread] = None + self._stop = threading.Event() + + def emit(self, event: RawUserEvent) -> None: + if self.status != RecorderState.RECORDING: + return + if event.from_inspector: + return + if self.on_event: + self.on_event(event) + + @abstractmethod + def start(self) -> None: ... + + @abstractmethod + def stop(self) -> None: ... + + def pause(self) -> None: + if self.status == RecorderState.RECORDING: + self.status = RecorderState.PAUSED + + def resume(self) -> None: + if self.status == RecorderState.PAUSED: + self.status = RecorderState.RECORDING + + @property + def capability(self) -> dict[str, str]: + return {} diff --git a/tools/ui_inspector/recorder/browser.py b/tools/ui_inspector/recorder/browser.py new file mode 100644 index 0000000..5a3c098 --- /dev/null +++ b/tools/ui_inspector/recorder/browser.py @@ -0,0 +1,740 @@ +"""Browser recorder for Inspector-owned Playwright sessions. + +Injects capture-phase listeners that do NOT preventDefault / stopPropagation +and do NOT mutate business DOM. Re-attaches after navigation. +""" + +from __future__ import annotations + +import json +import queue +import threading +import time +from typing import Any, Callable, Optional + +from .base import RawUserEvent, RecorderBase, RecorderState + +_LISTEN_SCRIPT = r""" +(() => { + if (window.__openclawUiInspectorRecorderInstalled) return; + window.__openclawUiInspectorRecorderInstalled = true; + window.__openclawUiInspectorEvents = window.__openclawUiInspectorEvents || []; + window.__openclawUiInspectorDocId = window.__openclawUiInspectorDocId || + (Date.now().toString(36) + "-" + Math.random().toString(36).slice(2)); + window.__openclawUiInspectorSeq = window.__openclawUiInspectorSeq || 0; + + function cssPath(el) { + if (!el || !el.tagName) return ""; + const esc = (value) => { + try { return CSS.escape(String(value)); } catch (e) { return String(value).replace(/[^a-zA-Z0-9_-]/g, "\\$&"); } + }; + const parts = []; + let cur = el; + for (let i = 0; cur && i < 12; i++) { + let part = cur.tagName.toLowerCase(); + if (cur.id) { part += "#" + esc(cur.id); parts.unshift(part); break; } + const cls = (cur.className && typeof cur.className === "string") + ? cur.className.trim().split(/\s+/).slice(0, 3).map(esc).join(".") + : ""; + if (cls) part += "." + cls; + parts.unshift(part); + cur = cur.parentElement; + } + return parts.join(" > "); + } + + function stableCss(el) { + if (!el || !el.getAttribute) return ""; + const quote = (value) => JSON.stringify(String(value)); + const ownId = el.id || ""; + if (ownId) return "#" + CSS.escape(ownId); + const testId = el.getAttribute("data-testid") || ""; + if (testId) return `[data-testid=${quote(testId)}]`; + const tag = (el.tagName || "").toLowerCase(); + let leaf = ""; + const href = el.getAttribute("href") || ""; + if (tag === "a" && href) leaf = `a[href=${quote(href)}]`; + else if (el.getAttribute("contenteditable") != null) leaf = `${tag || "*"}[contenteditable]`; + else if (el.getAttribute("name")) leaf = `${tag || "*"}[name=${quote(el.getAttribute("name"))}]`; + else if (el.getAttribute("aria-label")) leaf = `${tag || "*"}[aria-label=${quote(el.getAttribute("aria-label"))}]`; + else if (el.getAttribute("placeholder")) leaf = `${tag || "*"}[placeholder=${quote(el.getAttribute("placeholder"))}]`; + const stableClasses = (node) => { + if (!node || typeof node.className !== "string") return []; + return node.className.trim().split(/\s+/).filter((token) => { + if (!token || token.length < 4 || token.length > 64) return false; + const lower = token.toLowerCase(); + const semantic = /(button|btn|input|editor|control|action|submit|send|publish|post|play|pause|menu|option|tab|dialog|form|toolbar|container)/.test(lower); + const structured = /[-_]/.test(token) && !/^[a-z0-9_-]{6,14}$/i.test(token); + return semantic || structured; + }).slice(0, 2); + }; + if (!leaf) { + const classes = stableClasses(el); + leaf = tag + classes.map(cls => "." + CSS.escape(cls)).join(""); + } + let cur = el.parentElement; + const relative = [leaf]; + for (let i = 0; cur && i < 10; i++, cur = cur.parentElement) { + if (cur.id) return `#${CSS.escape(cur.id)} > ${relative.reverse().join(" > ")}`; + const ancestorTestId = cur.getAttribute?.("data-testid") || ""; + if (ancestorTestId) return `[data-testid=${quote(ancestorTestId)}] > ${relative.reverse().join(" > ")}`; + const curTag = (cur.tagName || "").toLowerCase(); + const classes = stableClasses(cur); + relative.push(curTag + classes.map(cls => "." + CSS.escape(cls)).join("")); + } + return ""; + } + + function editableValue(el) { + if (!el) return ""; + if (el.value != null) return String(el.value); + if (el.isContentEditable || el.getAttribute?.("contenteditable") != null) { + return String(el.innerText != null ? el.innerText : (el.textContent || "")); + } + return ""; + } + + function isEditable(el) { + if (!el || !el.getAttribute) return false; + const tag = (el.tagName || "").toLowerCase(); + return tag === "input" || tag === "textarea" || tag === "select" || + !!el.isContentEditable || el.getAttribute("contenteditable") != null; + } + + function editableRoot(el) { + let cur = el; + for (let i = 0; cur && i < 12; i++) { + if (cur.getAttribute && cur.getAttribute("contenteditable") != null) return cur; + const tag = (cur.tagName || "").toLowerCase(); + if (tag === "input" || tag === "textarea" || tag === "select") return cur; + cur = cur.parentElement; + } + return null; + } + + function semanticRole(el) { + if (!el || !el.getAttribute) return ""; + const explicit = el.getAttribute("role"); + if (explicit) return explicit; + const tag = (el.tagName || "").toLowerCase(); + if (el.isContentEditable) return "textbox"; + if (tag === "a") return "link"; + if (tag === "textarea") return "textbox"; + if (tag === "select") return "combobox"; + if (tag === "input") { + const type = (el.type || "text").toLowerCase(); + if (type === "checkbox") return "checkbox"; + if (type === "radio") return "radio"; + if (["button", "submit", "reset"].includes(type)) return "button"; + return "textbox"; + } + return tag; + } + + function actionableFromEvent(ev) { + const path = (ev && typeof ev.composedPath === "function") ? ev.composedPath() : []; + const nodes = path.length ? path : [ev && ev.target]; + let best = ev && ev.target; + let bestScore = -1; + for (const el of nodes) { + if (!el || !el.matches) continue; + const tag = (el.tagName || "").toLowerCase(); + const role = (el.getAttribute("role") || "").toLowerCase(); + const cls = typeof el.className === "string" ? el.className.toLowerCase() : ""; + let cursor = ""; + try { cursor = getComputedStyle(el).cursor || ""; } catch (e) {} + let score = 0; + if (el.matches("button,a,input,textarea,select,summary,label")) score = 100; + if (el.isContentEditable || el.getAttribute("contenteditable") != null) score = 98; + if (["button","link","textbox","combobox","checkbox","radio","menuitem","option","tab"].includes(role)) score = Math.max(score, 96); + else if (role) score = Math.max(score, 52); + if (el.getAttribute("data-testid") || el.getAttribute("aria-label")) score = Math.max(score, 90); + if (el.getAttribute("tabindex") != null) score = Math.max(score, 74); + if (cursor === "pointer") score = Math.max(score, 68); + // Many production apps use camelCase classes such as sendControl or + // playButton. Prefer that actionable wrapper over its inherited-cursor + // SVG/path child while still retaining hit_target for diagnostics. + if (tag.includes("-") || /(button|btn|submit|send|play|control|action|click)/i.test(cls)) score = Math.max(score, 72); + if (score > bestScore) { best = el; bestScore = score; } + } + return best; + } + + function snap(el) { + if (!el || !el.getAttribute) return {}; + const role = semanticRole(el); + let cursor = ""; + let rect = {}; + try { + cursor = getComputedStyle(el).cursor || ""; + const r = el.getBoundingClientRect(); + rect = {left:r.left, top:r.top, right:r.right, bottom:r.bottom, width:r.width, height:r.height}; + } catch (e) {} + const tag = el.tagName ? el.tagName.toLowerCase() : ""; + const css = cssPath(el); + const stable = stableCss(el); + let cssCount = null; + let stableCssCount = null; + try { cssCount = css ? document.querySelectorAll(css).length : null; } catch (e) {} + try { stableCssCount = stable ? document.querySelectorAll(stable).length : null; } catch (e) {} + const href = el.getAttribute("href") || ""; + let hrefCount = null; + try { + hrefCount = href ? Array.from(document.querySelectorAll("a[href]")).filter( + node => node.getAttribute("href") === href + ).length : null; + } catch (e) {} + const roleNames = ["button","link","textbox","combobox","checkbox","radio","menuitem","option","tab"]; + return { + role: role, + explicit_role: el.getAttribute("role") || "", + aria_label: el.getAttribute("aria-label") || "", + name: el.getAttribute("aria-label") || el.name || el.id || + ((el.matches?.("button,a,[role],label,summary") || roleNames.includes(role) || cursor === "pointer" || tag.includes("-")) + ? (el.innerText || el.textContent || "").trim().slice(0, 120) : ""), + text: (el.innerText || el.textContent || "").trim().slice(0, 120), + value: editableValue(el).slice(0, 500), + test_id: el.getAttribute("data-testid") || "", + name_attr: el.getAttribute("name") || "", + href: href, + href_count: hrefCount, + title: el.getAttribute("title") || "", + alt: el.getAttribute("alt") || "", + id: el.id || "", + placeholder: el.getAttribute("placeholder") || el.getAttribute("aria-placeholder") || "", + label: "", + input_type: (el.type || "").toLowerCase(), + tag: tag, + class_name: (typeof el.className === "string" ? el.className : "").slice(0, 300), + cursor: cursor, + rect: rect, + disabled: !!el.disabled || el.getAttribute("aria-disabled") === "true", + contenteditable: !!el.isContentEditable || el.getAttribute("contenteditable") != null, + contenteditable_attr: el.getAttribute("contenteditable") != null, + tabindex: el.getAttribute("tabindex"), + css: css, + css_count: cssCount, + stable_css: stable, + stable_css_count: stableCssCount, + fingerprint: [ + el.tagName?.toLowerCase() || "", el.id || "", + el.getAttribute("data-testid") || "", el.getAttribute("name") || "", + css + ].join("|"), + }; + } + + function ancestry(el) { + const out = []; + let cur = el; + for (let i = 0; cur && i < 12; i++) { + out.push(snap(cur)); + cur = cur.parentElement; + } + return out; + } + + function push(kind, payload) { + try { + const item = Object.assign({ + event_id: window.__openclawUiInspectorDocId + ":" + (++window.__openclawUiInspectorSeq), + document_id: window.__openclawUiInspectorDocId, + sequence: window.__openclawUiInspectorSeq, + kind: kind, + ts: Date.now(), + url: location.href, + title: document.title, + frame_url: location.href, + frame_name: window.name || "", + }, payload || {}); + window.__openclawUiInspectorEvents.push(item); + if (typeof window.__openclawUiInspectorEmit === "function") { + try { window.__openclawUiInspectorEmit(item); } catch (e) {} + } + if (window.__openclawUiInspectorEvents.length > 200) { + window.__openclawUiInspectorEvents.splice(0, 50); + } + } catch (e) {} + } + + document.addEventListener("click", (ev) => { + const hit = ev.target; + const action = actionableFromEvent(ev) || hit; + push("click", { + target: snap(action), + hit_target: snap(hit), + action_target: snap(action), + ancestry: ancestry(hit), + candidate_chain: ancestry(hit), + button: ev.button, + x: ev.clientX, + y: ev.clientY + }); + }, true); + + document.addEventListener("pointerdown", (ev) => { + const hit = ev.target; + push("pointerdown", { + target: snap(hit), + hit_target: snap(hit), + candidate_chain: ancestry(hit), + button: ev.button, + pointer_id: ev.pointerId, + pointer_type: ev.pointerType || "", + x: ev.clientX, + y: ev.clientY + }); + }, true); + + document.addEventListener("input", (ev) => { + if (ev.isComposing) return; + const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target; + push("input", { + target: snap(t), text: editableValue(t), input_event: ev.inputType || "input", + candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted + }); + }, true); + + document.addEventListener("change", (ev) => { + const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target; + push("input", { target: snap(t), text: editableValue(t), input_event: "change", + candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted }); + }, true); + + document.addEventListener("compositionend", (ev) => { + const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target; + push("input", { + target: snap(t), text: editableValue(t), input_event: "compositionend", + candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted + }); + }, true); + + document.addEventListener("paste", (ev) => { + const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target; + setTimeout(() => push("input", { + target: snap(t), text: editableValue(t), input_event: "paste", + candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted + }), 0); + }, true); + + document.addEventListener("keydown", (ev) => { + const special = ["Enter","Escape","Tab","Backspace","Delete"]; + const editing = isEditable(ev.target); + const mutatingEditKey = editing && (ev.key === "Backspace" || ev.key === "Delete"); + if (!mutatingEditKey && (special.indexOf(ev.key) >= 0 || (ev.key && ev.key.length > 1))) { + const t = actionableFromEvent(ev) || ev.target; + push("key", { key: ev.key, target: snap(t) }); + } + }, true); +})(); +""" + + +class BrowserRecorder(RecorderBase): + capability_map = { + "click": "implemented", + "fill": "implemented", + "press_key": "implemented", + "navigation": "partial", + } + + def __init__( + self, + *, + on_event: Optional[Callable[[RawUserEvent], None]] = None, + on_raw_event: Optional[Callable[[dict[str, Any]], None]] = None, + poll_sec: float = 0.25, + ) -> None: + super().__init__(on_event=on_event) + self.on_raw_event = on_raw_event + self.poll_sec = poll_sec + self._last_url = "" + self._last_urls_by_page: dict[str, str] = {} + self._installed_pages: set[int] = set() + self._bound_events: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=512) + self._seen_event_ids: set[str] = set() + self._seen_event_order: list[str] = [] + self._known_page_ids: set[str] = set() + self._pending_navigations: dict[str, dict[str, Any]] = {} + self._pending_pointerdowns: dict[str, dict[str, Any]] = {} + self._navigation_reorder_sec = 0.45 + self._pointer_fallback_sec = 0.35 + + @property + def capability(self) -> dict[str, str]: + return dict(self.capability_map) + + def start(self) -> None: + if self._thread and self._thread.is_alive(): + self.status = RecorderState.RECORDING + return + self._stop.clear() + self.status = RecorderState.RECORDING + try: + import kit_browser + + kit_browser.install_event_bridge(_LISTEN_SCRIPT, self._receive_bound_event) + initial_ids = kit_browser.call_on_each_open_page( + lambda page: kit_browser.page_id_for(page) + ) + self._known_page_ids.update( + str(page_id) for page_id in initial_ids if page_id + ) + except Exception: + # Polling remains as a compatibility fallback. + pass + self._thread = threading.Thread( + target=self._loop, name="ui-inspector-browser-recorder", daemon=True + ) + self._thread.start() + + def stop(self) -> None: + self._stop.set() + if self._thread and self._thread.is_alive(): + self._thread.join(timeout=2.0) + self._thread = None + # Disable the bridge before draining its local queue so an event that + # arrived just before Stop is not silently lost, while no new events + # can race with the drain. + try: + import kit_browser + + kit_browser.clear_event_bridge() + except Exception: + pass + if self.status == RecorderState.RECORDING: + try: + self._drain_bound_events() + self._flush_pending_pointerdowns(force=True) + self._flush_pending_navigations(force=True) + except Exception: + pass + self.status = RecorderState.STOPPED + + def _receive_bound_event(self, event: dict[str, Any]) -> None: + try: + self._bound_events.put_nowait(dict(event or {})) + except queue.Full: + try: + self._bound_events.get_nowait() + self._bound_events.put_nowait(dict(event or {})) + except (queue.Empty, queue.Full): + pass + + def _loop(self) -> None: + while not self._stop.is_set(): + if self.status == RecorderState.PAUSED: + time.sleep(0.1) + continue + if self.status != RecorderState.RECORDING: + time.sleep(0.05) + continue + try: + self._tick() + except Exception: + pass + time.sleep(self.poll_sec) + + def _tick(self) -> None: + import kit_browser + + self._drain_bound_events() + + # Install + drain on every open tab (detail may be a second tab). + payloads = kit_browser.call_on_each_open_page(self._install_and_drain) + if payloads: + for payload in payloads: + if not isinstance(payload, dict): + continue + page_id = str(payload.get("page_id") or "") + url = str(payload.get("url") or "") + previous_url = self._last_urls_by_page.get(page_id, "") if page_id else "" + if previous_url and url and url != previous_url: + self._schedule_navigation( + page_id=page_id, from_url=previous_url, to_url=url + ) + elif ( + page_id + and page_id not in self._known_page_ids + and self._known_page_ids + and url + and not url.startswith("about:") + ): + opener_id = str(payload.get("opener_page_id") or "") + self._schedule_navigation( + page_id=page_id, + from_url=self._last_urls_by_page.get(opener_id, ""), + to_url=url, + opener_page_id=opener_id, + new_page=True, + ) + if page_id and url: + self._last_urls_by_page[page_id] = url + self._known_page_ids.add(page_id) + for ev in payload.get("events") or []: + if isinstance(ev, dict) and not ev.get("page_id"): + ev["page_id"] = payload.get("page_id") or "" + self._emit_dom_event(ev) + + self._flush_pending_pointerdowns() + + # Navigation is intentionally emitted after both bridge and polling + # queues have been drained. This preserves click -> navigation order + # even when the context bridge is unavailable and polling is used. + self._flush_pending_navigations() + + def _drain_bound_events(self) -> None: + while True: + try: + self._emit_dom_event(self._bound_events.get_nowait()) + except queue.Empty: + break + + def _schedule_navigation( + self, + *, + page_id: str, + from_url: str, + to_url: str, + opener_page_id: str = "", + new_page: bool = False, + ) -> None: + self._pending_navigations[page_id] = { + "due": time.monotonic() + self._navigation_reorder_sec, + "from": from_url, + "to": to_url, + "page_id": page_id, + "opener_page_id": opener_page_id, + "new_page": new_page, + } + + def _flush_pending_navigations(self, *, force: bool = False) -> None: + now = time.monotonic() + ready = [ + page_id + for page_id, item in self._pending_navigations.items() + if force or float(item.get("due") or 0.0) <= now + ] + for page_id in ready: + self._flush_pending_pointerdowns(force=True, page_id=page_id) + item = self._pending_navigations.pop(page_id, None) or {} + self.emit( + RawUserEvent( + kind="navigation", + surface="browser", + url=str(item.get("to") or ""), + meta={ + "from": item.get("from") or "", + "to": item.get("to") or "", + "page_id": page_id, + "opener_page_id": item.get("opener_page_id") or "", + "new_page": bool(item.get("new_page")), + }, + ) + ) + + def _install_and_drain(self, page: Any) -> dict[str, Any]: + """Runs on Playwright worker thread (once per open tab).""" + try: + page.evaluate(_LISTEN_SCRIPT) + except Exception: + pass + # Also try frames + try: + for frame in page.frames: + try: + frame.evaluate(_LISTEN_SCRIPT) + except Exception: + pass + except Exception: + pass + events = [] + try: + events = page.evaluate( + """() => { + const e = window.__openclawUiInspectorEvents || []; + window.__openclawUiInspectorEvents = []; + return e; + }""" + ) or [] + except Exception: + events = [] + url = "" + try: + url = page.url or "" + except Exception: + url = "" + page_id = "" + opener_page_id = "" + try: + import kit_browser + + page_id = kit_browser.page_id_for(page) + try: + opener = page.opener() + if opener is not None: + opener_page_id = kit_browser.page_id_for(opener) + except Exception: + pass + except Exception: + pass + return { + "url": url, + "page_id": page_id, + "opener_page_id": opener_page_id, + "events": events, + } + + def _emit_dom_event(self, ev: dict[str, Any]) -> None: + event_id = str(ev.get("event_id") or "") + if event_id: + if event_id in self._seen_event_ids: + return + self._seen_event_ids.add(event_id) + self._seen_event_order.append(event_id) + if len(self._seen_event_order) > 2000: + old = self._seen_event_order.pop(0) + self._seen_event_ids.discard(old) + if self.on_raw_event: + try: + self.on_raw_event(dict(ev)) + except Exception: + pass + kind = str(ev.get("kind") or "") + hit_target = dict(ev.get("hit_target") or ev.get("target") or {}) + candidate_chain = list(ev.get("candidate_chain") or ev.get("ancestry") or []) + if not candidate_chain: + candidate_chain = [hit_target] + action_hint = dict(ev.get("action_target") or {}) + if action_hint and action_hint != hit_target: + candidate_chain.append(action_hint) + try: + from core.policies.target_scoring import resolve_browser_target + + target, target_scores = resolve_browser_target( + hit_target, + candidate_chain, + x=int(ev.get("x")) if ev.get("x") is not None else None, + y=int(ev.get("y")) if ev.get("y") is not None else None, + ) + except Exception: + target = dict(ev.get("action_target") or ev.get("target") or hit_target) + target_scores = [] + if candidate_chain: + target["parent_chain"] = candidate_chain + frame = { + "url": ev.get("frame_url") or "", + "name": ev.get("frame_name") or "", + "id": ev.get("frame_id") or "", + } + if kind in {"pointerdown", "click"}: + raw = RawUserEvent( + kind="click", + surface="browser", + target=target, + frame=frame, + url=str(ev.get("url") or ""), + x=int(ev.get("x")) if ev.get("x") is not None else None, + y=int(ev.get("y")) if ev.get("y") is not None else None, + meta={ + "button": ev.get("button"), + "event_id": event_id, + "document_id": ev.get("document_id"), + "page_id": ev.get("page_id"), + "frame_id": ev.get("frame_id"), + "sequence": ev.get("sequence"), + "source_ts_ms": ev.get("ts"), + "hit_target": ev.get("hit_target") or {}, + "action_target": ev.get("action_target") or target, + "target_scores": target_scores, + "captured_from": kind, + }, + ) + if kind == "pointerdown": + key = self._pointer_key(ev) + self._pending_pointerdowns[key] = { + "due": time.monotonic() + self._pointer_fallback_sec, + "event": raw, + "page_id": str(ev.get("page_id") or ""), + } + return + self._cancel_pointerdown_for_click(ev) + self.emit(raw) + elif kind == "input": + self.emit( + RawUserEvent( + kind="input", + surface="browser", + text=str(ev.get("text") or target.get("value") or ""), + target=target, + frame=frame, + url=str(ev.get("url") or ""), + meta={ + "input_type": target.get("input_type") or "", + "event_id": event_id, + "document_id": ev.get("document_id"), + "page_id": ev.get("page_id"), + "frame_id": ev.get("frame_id"), + "sequence": ev.get("sequence"), + "source_ts_ms": ev.get("ts"), + "input_event": ev.get("input_event") or "input", + "is_trusted": bool(ev.get("is_trusted")), + }, + ) + ) + elif kind == "key": + self.emit( + RawUserEvent( + kind="key", + surface="browser", + key=str(ev.get("key") or ""), + target=target, + frame=frame, + url=str(ev.get("url") or ""), + meta={ + "event_id": event_id, + "document_id": ev.get("document_id"), + "page_id": ev.get("page_id"), + "frame_id": ev.get("frame_id"), + "sequence": ev.get("sequence"), + "source_ts_ms": ev.get("ts"), + }, + ) + ) + + @staticmethod + def _pointer_key(ev: dict[str, Any]) -> str: + return "|".join( + str(ev.get(key) or "") + for key in ("page_id", "document_id", "pointer_id", "button") + ) + + def _cancel_pointerdown_for_click(self, ev: dict[str, Any]) -> None: + exact = self._pointer_key(ev) + if exact in self._pending_pointerdowns: + self._pending_pointerdowns.pop(exact, None) + return + page_id = str(ev.get("page_id") or "") + button = str(ev.get("button") or "") + for key, item in list(self._pending_pointerdowns.items()): + event = item.get("event") + meta = getattr(event, "meta", {}) or {} + if str(meta.get("page_id") or "") == page_id and str(meta.get("button") or "") == button: + self._pending_pointerdowns.pop(key, None) + return + + def _flush_pending_pointerdowns( + self, *, force: bool = False, page_id: str = "" + ) -> None: + now = time.monotonic() + ready = [ + key for key, item in self._pending_pointerdowns.items() + if (not page_id or item.get("page_id") == page_id) + and (force or float(item.get("due") or 0.0) <= now) + ] + for key in ready: + item = self._pending_pointerdowns.pop(key, None) or {} + event = item.get("event") + if event is not None: + event.meta["pointerdown_fallback"] = True + self.emit(event) diff --git a/tools/ui_inspector/recorder/input_aggregator.py b/tools/ui_inspector/recorder/input_aggregator.py new file mode 100644 index 0000000..adc9bee --- /dev/null +++ b/tools/ui_inspector/recorder/input_aggregator.py @@ -0,0 +1,234 @@ +"""Aggregate keystrokes / value changes into fill/type + press_key.""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Any, Optional + +from redaction import redact_input_payload + +# Keys recorded as independent press_key actions. +_FUNCTION_KEYS = { + "Enter", + "Escape", + "Tab", + "Backspace", + "Delete", + "ArrowUp", + "ArrowDown", + "ArrowLeft", + "ArrowRight", + "Home", + "End", + "PageUp", + "PageDown", + "F1", + "F2", + "F3", + "F4", + "F5", + "F6", + "F7", + "F8", + "F9", + "F10", + "F11", + "F12", +} + + +@dataclass +class AggregatedInput: + action_type: str # fill | type | press_key + text: str = "" + key: str = "" + target: dict[str, Any] = field(default_factory=dict) + sensitive: bool = False + input_payload: dict[str, Any] = field(default_factory=dict) + started_at: float = 0.0 + ended_at: float = 0.0 + capture_context: dict[str, Any] = field(default_factory=dict) + raw_event: dict[str, Any] = field(default_factory=dict) + raw_event_ids: list[str] = field(default_factory=list) + + +class InputAggregator: + """ + Do not emit one Action per key. + Observe focus element value changes; flush after idle gap or function key. + """ + + def __init__(self, *, idle_sec: float = 0.8) -> None: + self.idle_sec = idle_sec + self._buf_text = "" + self._target: dict[str, Any] = {} + self._started: float = 0.0 + self._last: float = 0.0 + self._input_type = "" + self._field_name = "" + self._capture_context: dict[str, Any] = {} + self._raw_event: dict[str, Any] = {} + self._raw_event_ids: list[str] = [] + self._target_identity = "" + + @staticmethod + def _stable_target_identity( + target: dict[str, Any], capture_context: dict[str, Any] + ) -> str: + from core.policies.element_identity import element_identity_key + + return element_identity_key( + target or {}, + surface=str(capture_context.get("surface") or "windows"), + page_url=str(capture_context.get("url") or ""), + frame_name=str(capture_context.get("frame_name") or ""), + ) + + @staticmethod + def _event_id(raw_event: Optional[dict[str, Any]]) -> str: + return str(((raw_event or {}).get("meta") or {}).get("event_id") or "") + + def on_key( + self, + *, + key: str, + text: str = "", + target: Optional[dict[str, Any]] = None, + input_type: str = "", + field_name: str = "", + capture_context: Optional[dict[str, Any]] = None, + raw_event: Optional[dict[str, Any]] = None, + value_event: bool = False, + now: Optional[float] = None, + ) -> list[AggregatedInput]: + now = time.monotonic() if now is None else now + out: list[AggregatedInput] = [] + k = key or "" + if k in _FUNCTION_KEYS or (len(k) > 1 and k.startswith("F") and k[1:].isdigit()): + out.extend(self.flush(now=now)) + payload, sens = redact_input_payload( + {"key": k}, + field_name=field_name or self._field_name, + input_type=input_type or self._input_type, + ) + out.append( + AggregatedInput( + action_type="press_key", + key=k, + target=dict(target or self._target or {}), + sensitive=sens, + input_payload=payload, + started_at=now, + ended_at=now, + capture_context=dict(capture_context or self._capture_context or {}), + raw_event=dict(raw_event or {}), + raw_event_ids=( + [self._event_id(raw_event)] if self._event_id(raw_event) else [] + ), + ) + ) + return out + + # Printable / value change + if target: + # Focus changed → flush previous + next_context = dict(capture_context or self._capture_context or {}) + next_identity = self._stable_target_identity(target, next_context) + if self._target_identity and next_identity != self._target_identity: + out.extend(self.flush(now=now)) + self._target = dict(target) + self._target_identity = next_identity + if input_type: + self._input_type = input_type + if field_name: + self._field_name = field_name + if capture_context: + self._capture_context = dict(capture_context) + if raw_event: + self._raw_event = dict(raw_event) + event_id = self._event_id(raw_event) + if event_id and event_id not in self._raw_event_ids: + self._raw_event_ids.append(event_id) + + chunk = text if text else (k if len(k) == 1 else "") + if not chunk and not text and not value_event: + return out + + if not self._started: + self._started = now + # If observer sends full value, replace buffer; else append char + if value_event: + self._buf_text = text + elif text and len(text) >= len(self._buf_text): + self._buf_text = text + else: + self._buf_text += chunk + self._last = now + return out + + def on_value_change( + self, + *, + value: str, + target: Optional[dict[str, Any]] = None, + input_type: str = "", + field_name: str = "", + capture_context: Optional[dict[str, Any]] = None, + raw_event: Optional[dict[str, Any]] = None, + now: Optional[float] = None, + ) -> list[AggregatedInput]: + return self.on_key( + key="", + text=value, + target=target, + input_type=input_type, + field_name=field_name, + capture_context=capture_context, + raw_event=raw_event, + value_event=True, + now=now, + ) + + def flush(self, *, now: Optional[float] = None) -> list[AggregatedInput]: + now = ( + (time.time() if self._last > 100_000_000 else time.monotonic()) + if now is None else now + ) + if not self._buf_text and not self._started: + return [] + payload, sens = redact_input_payload( + {"text": self._buf_text, "name": self._field_name}, + field_name=self._field_name, + input_type=self._input_type, + ) + item = AggregatedInput( + action_type="fill", + text=str(payload.get("text") or ""), + target=dict(self._target or {}), + sensitive=sens, + input_payload=payload, + started_at=self._started or now, + ended_at=now, + capture_context=dict(self._capture_context or {}), + raw_event=dict(getattr(self, "_raw_event", {}) or {}), + raw_event_ids=list(self._raw_event_ids), + ) + self._buf_text = "" + self._started = 0.0 + self._last = 0.0 + self._input_type = "" + self._capture_context = {} + self._raw_event = {} + self._raw_event_ids = [] + # keep target for next session + return [item] + + def maybe_flush_idle(self, *, now: Optional[float] = None) -> list[AggregatedInput]: + now = ( + (time.time() if self._last > 100_000_000 else time.monotonic()) + if now is None else now + ) + if self._last and (now - self._last) >= self.idle_sec and self._started: + return self.flush(now=now) + return [] diff --git a/tools/ui_inspector/recorder/manager.py b/tools/ui_inspector/recorder/manager.py new file mode 100644 index 0000000..c7d4c96 --- /dev/null +++ b/tools/ui_inspector/recorder/manager.py @@ -0,0 +1,1220 @@ +"""Action recording manager: before → action → after, dual-write v1/v2.""" + +from __future__ import annotations + +import threading +import time +from datetime import datetime +from pathlib import Path +import html +import re +from typing import Any, Callable, Optional + +from locators.catalog import ElementCatalogService +from models.action import ACTION_CAPABILITY, ActionFlow, ActionRecord +from models.serialize import to_dict, write_model_json +from redaction import redact_input_payload, sanitize_artifact +from snapshot_service import SnapshotService +from state_diff import ( + changes_to_dicts, + diff_states, + postconditions_to_dicts, + propose_postconditions, +) +from core.policies.element_identity import element_identity_key + +from .base import RawUserEvent, RecorderState +from .input_aggregator import InputAggregator + + +def _now_iso() -> str: + return datetime.now().astimezone().isoformat() + + +class ActionRecordingManager: + """ + Central coordinator. + + - Action is the flow spine + - States are before/after evidence (fingerprint-deduped) + - Dual-writes optional v1 step via callback + """ + + def __init__( + self, + flow_dir: Path, + *, + flow_name: str = "", + flow_id: str = "", + surface: str = "windows", + recording_mode: str = "auto", + dump_fn: Optional[Callable[[], Any]] = None, + write_v1_step: Optional[Callable[..., Any]] = None, + stability_timeout_sec: float = 3.0, + save_sensitive_screenshots: bool = False, + defer_action_evidence: bool = False, + on_action: Optional[Callable[[ActionRecord], None]] = None, + on_action_update: Optional[Callable[[ActionRecord, str], None]] = None, + ) -> None: + self.flow_dir = Path(flow_dir) + self.actions_dir = self.flow_dir / "actions" + self.actions_dir.mkdir(parents=True, exist_ok=True) + self.snapshots = SnapshotService( + self.flow_dir, + save_sensitive_screenshots=save_sensitive_screenshots, + ) + self.catalog = ElementCatalogService(self.flow_dir, flow_id=flow_id or flow_name) + self.aggregator = InputAggregator(idle_sec=8.0 if surface == "browser" else 0.8) + self.dump_fn = dump_fn + self.defer_action_evidence = bool(defer_action_evidence) + self.on_action = on_action + self.on_action_update = on_action_update + self.write_v1_step = write_v1_step + self.stability_timeout_sec = stability_timeout_sec + self._lock = threading.RLock() + self._seq = 0 + self.status = RecorderState.IDLE + self.last_action: Optional[ActionRecord] = None + self.last_warning: str = "" + self.action_flow = ActionFlow( + flow_id=flow_id or self.flow_dir.name, + flow_name=flow_name or self.flow_dir.name, + recording_mode=recording_mode, + surface=surface, + metadata={"action_capability": dict(ACTION_CAPABILITY)}, + ) + self._debounce_until = 0.0 + self._debounce_sec = 0.35 + self._last_click_signature = "" + self._last_click_monotonic = 0.0 + self._actions_by_id: dict[str, ActionRecord] = {} + self._last_fill: dict[str, tuple[str, float]] = {} + self._last_text_target: dict[str, Any] = {} + + def _dump_for_context(self, context: Optional[dict[str, Any]] = None) -> Any: + if not self.dump_fn: + return None + ctx = dict(context or {}) + kwargs = { + key: ctx.get(key) + for key in ("page_id", "document_id", "frame_id") + if ctx.get(key) + } + if kwargs: + try: + return self.dump_fn(**kwargs) + except TypeError as exc: + if "unexpected keyword" not in str(exc) and "positional" not in str(exc): + raise + return self.dump_fn() + + @staticmethod + def _event_context(event: RawUserEvent) -> dict[str, Any]: + meta = dict(event.meta or {}) + return { + "surface": event.surface or "", + "page_id": meta.get("page_id") or "", + "document_id": meta.get("document_id") or "", + "frame_id": meta.get("frame_id") or "", + "url": event.url or "", + "frame_url": (event.frame or {}).get("url") or "", + "frame_name": (event.frame or {}).get("name") or "", + "source_ts_ms": meta.get("source_ts_ms"), + "event_id": meta.get("event_id") or "", + "sequence": meta.get("sequence"), + "timestamp": event.timestamp, + "input_event": meta.get("input_event") or "", + "is_trusted": meta.get("is_trusted"), + } + + @staticmethod + def _captured_live_counts(target: dict[str, Any]) -> Optional[dict[str, int]]: + """Use uniqueness measured synchronously in the event-origin document.""" + counts: dict[str, int] = {} + for strategy, field in ( + ("css", "css_count"), + ("stable_css", "stable_css_count"), + ("href", "href_count"), + ): + value = (target or {}).get(field) + if value is not None: + counts[strategy] = int(value) + return counts or None + + def _stable_capture( + self, context: dict[str, Any], *, reason: str, timeout: float + ): + return self.snapshots.wait_until_stable( + lambda: self._dump_for_context(context), + timeout_sec=timeout, + change_reason=reason, + ) + + def _action_capture(self, context: dict[str, Any], *, reason: str): + """Capture one post-action observation without blocking on two dumps. + + Browser user events can arrive faster than a full page dump. A second + stability poll made the FIFO lag by several seconds and contaminated + earlier actions with later input. The action stream keeps ordering; + quiet/checkpoint capture remains available for deliberately stable UI. + """ + import time + + time.sleep(0.06) + dump = self._dump_for_context(context) + return self.snapshots.capture_from_dump( + dump, change_reason=reason, force_persist=True + ) + + @staticmethod + def _enrich_weak_browser_target(target: dict[str, Any], before) -> dict[str, Any]: + """Attach a semantic hint without discarding the physical hit target.""" + out = dict(target or {}) + role = str(out.get("role") or "").lower() + if role not in {"", "path", "svg", "img", "span", "div"}: + return out + if str(out.get("name") or out.get("text") or "").strip(): + return out + scope_text = " ".join( + [str(out.get("css") or "")] + + [ + " ".join( + str((ancestor or {}).get(key) or "") + for key in ("id", "class_name", "css") + ) + for ancestor in (out.get("parent_chain") or []) + ] + ).lower() + form_hints = ( + "comment", "editor", "input", "form", "reply", "message", "chat", + "评论", "输入", "回复", "消息", + ) + if not any(token in scope_text for token in form_hints): + return out + action_names = { + "submit", "send", "publish", "post", "reply", "comment", + "提交", "发送", "发布", "回复", "评论", + "送信", "投稿", "公開", "보내기", "게시", "enviar", "publicar", + "envoyer", "publier", "senden", "veröffentlichen", + } + candidates = [] + for row in list(getattr(before, "locatable", None) or []): + row_role = str(row.get("role") or row.get("control_type") or "").lower() + row_name = str(row.get("name") or row.get("value") or row.get("text") or "").strip() + if row_role in {"button", "menuitem", "link"} and row_name.lower() in action_names: + candidates.append(row) + unique = { + (str(row.get("role") or row.get("control_type") or ""), str(row.get("name") or row.get("value") or row.get("text") or "")) + for row in candidates + } + if len(unique) == 1: + semantic_role, semantic_name = next(iter(unique)) + out["semantic_role"] = semantic_role + out["semantic_name"] = semantic_name + out["semantic_inference"] = "form_scope_unique_action" + out["semantic_confidence"] = 0.72 + return out + + def start(self, *, entry_dump: Optional[Any] = None) -> ActionRecord | None: + with self._lock: + self.status = RecorderState.RECORDING + self.action_flow.status = "open" + self.action_flow.created_at = _now_iso() + dump = entry_dump + if dump is None and self.dump_fn: + try: + dump = self._dump_for_context() + except Exception as exc: + self.last_warning = f"entry dump failed: {exc}" + dump = None + if dump is not None: + st = self.snapshots.capture_from_dump( + dump, change_reason="entry", force_persist=True + ) + self.action_flow.entry_state_id = st.id + self.action_flow.final_state_id = st.id + if st.id not in self.action_flow.states: + self.action_flow.states.append(st.id) + if self.snapshots.latest_observation: + self.action_flow.observations.append( + self.snapshots.latest_observation.id + ) + # Optional v1 step for entry checkpoint + if self.write_v1_step: + try: + self.write_v1_step(dump, label_hint="entry", change_reason="entry") + except Exception: + pass + self._persist_flow() + return None + + def pause(self) -> None: + self.status = RecorderState.PAUSED + + def resume(self) -> None: + if self.status == RecorderState.PAUSED: + self.status = RecorderState.RECORDING + + def stop(self) -> ActionFlow: + with self._lock: + for item in self.aggregator.flush(): + self._commit_fill(item) + self.status = RecorderState.STOPPED + self.action_flow.status = "ended" + self.action_flow.ended_at = _now_iso() + if self.snapshots.latest: + self.action_flow.final_state_id = self.snapshots.latest.id + self._persist_flow() + self.catalog.save() + return self.action_flow + + def handle_raw_event(self, event: RawUserEvent) -> Optional[ActionRecord]: + if self.status != RecorderState.RECORDING: + return None + if event.from_inspector: + return None + with self._lock: + kind = event.kind + if kind in {"mouse_down", "click"}: + return self._handle_click(event) + if kind in {"key", "input", "value_change"}: + return self._handle_input(event) + if kind == "navigation": + return self._handle_navigation(event) + if kind == "checkpoint": + return self.record_checkpoint(meta=event.meta) + return None + + def record_checkpoint(self, *, meta: Optional[dict[str, Any]] = None) -> Optional[ActionRecord]: + with self._lock: + before = self.snapshots.latest + before_obs = self.snapshots.latest_observation + dump = None + if self.dump_fn: + try: + dump = self._dump_for_context(meta) + except Exception as exc: + return self._failed_action( + "checkpoint", error=str(exc), before=before + ) + after = self.snapshots.capture_from_dump( + dump, change_reason="checkpoint", force_persist=True + ) + return self._finalize_action( + action_type="checkpoint", + before=before, + after=after, + target={}, + input_payload={}, + raw_event={"meta": meta or {}}, + confidence=0.9, + before_observation_id=before_obs.id if before_obs else None, + after_observation_id=( + self.snapshots.latest_observation.id + if self.snapshots.latest_observation else None + ), + ) + + def _handle_click(self, event: RawUserEvent) -> Optional[ActionRecord]: + import time + + now = time.monotonic() + context = self._event_context(event) + target = dict(event.target or {}) + click_identity = element_identity_key( + target, + surface=event.surface, + page_url=str(context.get("url") or ""), + frame_name=str(context.get("frame_name") or ""), + ) + signature = "|".join( + (click_identity, str(event.button or ""), str(event.x), str(event.y)) + ) + if ( + signature == self._last_click_signature + and now - self._last_click_monotonic < self._debounce_sec + ): + return None + self._last_click_signature = signature + self._last_click_monotonic = now + # Flush pending input before click + for item in self.aggregator.flush(): + self._commit_fill(item) + + before = self.snapshots.latest + before_obs = self.snapshots.latest_observation + if event.surface == "browser": + target = self._enrich_weak_browser_target(target, before) + element = None + live_counts = self._captured_live_counts(target) + if ( + not self.defer_action_evidence + and event.surface == "browser" and context.get("page_id") and target + ): + try: + import kit_browser + + live_counts = kit_browser.validate_element_on_page( + page_id=str(context.get("page_id") or ""), + element=target, + frame=event.frame or None, + ) + except Exception as exc: + self.last_warning = f"live locator validation failed: {exc}" + if target: + try: + element = self.catalog.upsert_from_snapshot( + target, + surface=event.surface, + window_scope=target.get("window_scope") or {}, + rows=(before.locatable if before else None), + frame=event.frame or None, + live_counts=live_counts, + ) + except Exception as exc: + self.last_warning = f"catalog upsert failed: {exc}" + + after = before + status = "ok" + err = "" + if self.dump_fn and not self.defer_action_evidence: + try: + after = self._action_capture(context, reason="after_click") + except Exception as exc: + status = "partial" + err = str(exc) + try: + dump = self._dump_for_context(context) + after = self.snapshots.capture_from_dump( + dump, change_reason="after_click_partial", force_persist=True + ) + except Exception as exc2: + status = "failed" + err = f"{err}; {exc2}" + + act = self._finalize_action( + action_type="click", + before=before, + after=after, + target=target, + target_element_id=element.id if element else None, + input_payload={}, + raw_event=to_dict(event) if hasattr(event, "__dataclass_fields__") else dict(event.__dict__), + status=status, + confidence=0.8 if status == "ok" else 0.4, + evidence={"error": err} if err else {}, + started_at=event.timestamp, + before_observation_id=before_obs.id if before_obs else None, + after_observation_id=( + self.snapshots.latest_observation.id + if self.snapshots.latest_observation else None + ), + ) + if self.write_v1_step and after is not None: + # Dual-write one v1 step per meaningful action (not heartbeat) + try: + # Reconstruct a minimal dump-like for v1 from after state paths is hard; + # caller should pass dump through write_v1_step if needed. + pass + except Exception: + pass + return act + + def _handle_input(self, event: RawUserEvent) -> Optional[ActionRecord]: + key = event.key or (event.meta or {}).get("key") or "" + text = event.text or (event.meta or {}).get("text") or (event.meta or {}).get("value") or "" + common = { + "target": event.target, + "input_type": str((event.meta or {}).get("input_type") or ""), + "field_name": str((event.target or {}).get("name") or ""), + "capture_context": self._event_context(event), + "raw_event": to_dict(event), + } + source_ms = (event.meta or {}).get("source_ts_ms") + event_clock = ( + float(source_ms) / 1000.0 if source_ms is not None else time.monotonic() + ) + if event.kind in {"input", "value_change"}: + items = self.aggregator.on_value_change(value=text, now=event_clock, **common) + else: + items = self.aggregator.on_key(key=key, text=text, now=event_clock, **common) + items.extend(self.aggregator.maybe_flush_idle(now=event_clock)) + last = None + for item in items: + if item.action_type == "press_key": + last = self._commit_press_key(item) + else: + last = self._commit_fill(item) + return last + + def _handle_navigation(self, event: RawUserEvent) -> Optional[ActionRecord]: + before = self.snapshots.latest + before_obs = self.snapshots.latest_observation + context = self._event_context(event) + after = before + if self.dump_fn and not self.defer_action_evidence: + try: + after = self._action_capture(context, reason="navigation") + except Exception: + pass + return self._finalize_action( + action_type="navigation", + before=before, + after=after, + target={}, + input_payload={"url": event.url}, + raw_event={"url": event.url, "meta": event.meta}, + confidence=0.7, + started_at=event.timestamp, + before_observation_id=before_obs.id if before_obs else None, + after_observation_id=(self.snapshots.latest_observation.id if self.snapshots.latest_observation else None), + ) + + def _commit_fill(self, item) -> Optional[ActionRecord]: + if ( + item.target + and (self.action_flow.surface or "") == "windows" + and not self._is_oversized_windows_target(item.target) + ): + self._last_text_target = dict(item.target) + page_url = str(item.capture_context.get("url") or "") + identity = element_identity_key( + item.target or {}, + surface=self.action_flow.surface or "windows", + page_url=page_url, + frame_name=str(item.capture_context.get("frame_name") or ""), + ) + text_value = str((item.input_payload or {}).get("text") or "") + previous = self._last_fill.get(identity) + source_ms = item.capture_context.get("source_ts_ms") + now_marker = float(source_ms) / 1000.0 if source_ms else time.time() + if previous and previous[0] == text_value and now_marker - previous[1] <= 15.0: + return None + self._last_fill[identity] = (text_value, now_marker) + before = self.snapshots.latest + before_obs = self.snapshots.latest_observation + after = before + if self.dump_fn and not self.defer_action_evidence: + try: + after = self._action_capture( + item.capture_context, reason="after_fill" + ) + except Exception: + pass + element = None + frame = { + "url": item.capture_context.get("frame_url") or "", + "name": item.capture_context.get("frame_name") or "", + } + live_counts = self._captured_live_counts(item.target or {}) + if ( + not self.defer_action_evidence + and item.capture_context.get("page_id") and item.target + ): + try: + import kit_browser + + live_counts = kit_browser.validate_element_on_page( + page_id=str(item.capture_context.get("page_id") or ""), + element=item.target, + frame=frame, + ) + except Exception as exc: + self.last_warning = f"live fill locator validation failed: {exc}" + if item.target: + try: + element = self.catalog.upsert_from_snapshot( + item.target, + surface=self.action_flow.surface or "windows", + window_scope=item.target.get("window_scope") or {}, + rows=(before.locatable if before else None), + frame=frame, + live_counts=live_counts, + ) + except Exception: + pass + return self._finalize_action( + action_type="fill", + before=before, + after=after, + target=item.target, + target_element_id=element.id if element else None, + input_payload=item.input_payload, + sensitive=item.sensitive, + confidence=0.75, + raw_event=item.raw_event or { + "url": item.capture_context.get("url") or "", + "meta": dict(item.capture_context or {}), + }, + raw_event_ids=item.raw_event_ids, + started_at=str(item.capture_context.get("timestamp") or _now_iso()), + before_observation_id=before_obs.id if before_obs else None, + after_observation_id=(self.snapshots.latest_observation.id if self.snapshots.latest_observation else None), + ) + + def _commit_press_key(self, item) -> ActionRecord: + target = dict(item.target or {}) + if ( + (self.action_flow.surface or "") == "windows" + and self._is_oversized_windows_target(target) + and self._last_text_target + and self._same_windows_target_scope(target, self._last_text_target) + ): + target = dict(self._last_text_target) + before = self.snapshots.latest + before_obs = self.snapshots.latest_observation + after = before + if self.dump_fn and not self.defer_action_evidence: + try: + after = self._action_capture( + item.capture_context, reason="after_key" + ) + except Exception: + pass + element = None + frame = { + "url": item.capture_context.get("frame_url") or "", + "name": item.capture_context.get("frame_name") or "", + } + live_counts = self._captured_live_counts(target) + if target: + try: + element = self.catalog.upsert_from_snapshot( + target, + surface=self.action_flow.surface or "windows", + window_scope=target.get("window_scope") or {}, + rows=(before.locatable if before else None), + frame=frame, + live_counts=live_counts, + ) + except Exception as exc: + self.last_warning = f"key target catalog upsert failed: {exc}" + return self._finalize_action( + action_type="press_key", + before=before, + after=after, + target=target, + target_element_id=element.id if element else None, + input_payload=item.input_payload, + sensitive=item.sensitive, + confidence=0.7, + raw_event=item.raw_event or { + "kind": "key", + "surface": self.action_flow.surface or "", + "key": item.key, + "target": target, + "frame": frame, + "url": item.capture_context.get("url") or "", + "meta": dict(item.capture_context or {}), + }, + raw_event_ids=item.raw_event_ids, + started_at=str(item.capture_context.get("timestamp") or _now_iso()), + before_observation_id=before_obs.id if before_obs else None, + after_observation_id=(self.snapshots.latest_observation.id if self.snapshots.latest_observation else None), + ) + + def _failed_action( + self, action_type: str, *, error: str, before + ) -> ActionRecord: + return self._finalize_action( + action_type=action_type, + before=before, + after=before, + target={}, + input_payload={}, + status="failed", + confidence=0.1, + evidence={"error": error}, + ) + + def _finalize_action( + self, + *, + action_type: str, + before, + after, + target: dict[str, Any], + input_payload: dict[str, Any], + target_element_id: Optional[str] = None, + sensitive: bool = False, + raw_event: Optional[dict[str, Any]] = None, + raw_event_ids: Optional[list[str]] = None, + status: str = "ok", + confidence: float = 0.5, + evidence: Optional[dict[str, Any]] = None, + before_observation_id: Optional[str] = None, + after_observation_id: Optional[str] = None, + started_at: Optional[str] = None, + ) -> ActionRecord: + self._seq += 1 + payload, sens2 = redact_input_payload(input_payload or {}) + sensitive = sensitive or sens2 + clean_target = sanitize_artifact(target or {}, sensitive_hint=sensitive) + clean_raw_event = sanitize_artifact(raw_event or {}, sensitive_hint=sensitive) + + before_id = before.id if before else None + after_id = after.id if after else before_id + if self.defer_action_evidence and action_type != "checkpoint": + # A slow eventual dump must not be presented as an event-time + # after-state. The evidence coordinator attaches it explicitly. + after_id = None + after_observation_id = None + if after and after.id not in self.action_flow.states: + self.action_flow.states.append(after.id) + if before and before.id not in self.action_flow.states: + self.action_flow.states.append(before.id) + + observed = [] + proposed = [] + if before and after: + b = self.snapshots.to_summary_dict(before) + a = self.snapshots.to_summary_dict(after) + changes = diff_states(b, a) + observed = changes_to_dicts(changes) + proposed = postconditions_to_dicts(propose_postconditions(changes)) + + # Attach locator uniqueness hint for UI + match_count = None + unique = None + primary_expr = "" + if target_element_id: + for el in self.catalog.catalog.elements: + if el.id == target_element_id and el.primary_locator: + match_count = el.primary_locator.actual_count + unique = el.primary_locator.unique + primary_expr = el.primary_locator.expression + break + if target_element_id not in self.action_flow.elements: + self.action_flow.elements.append(target_element_id) + + if raw_event_ids is None: + event_id = str(((clean_raw_event or {}).get("meta") or {}).get("event_id") or "") + raw_event_ids = [event_id] if event_id else [] + act = ActionRecord( + sequence=self._seq, + action_type=action_type, + started_at=started_at or _now_iso(), + ended_at=_now_iso(), + surface=self.action_flow.surface or "", + target_element_id=target_element_id, + target_snapshot=dict(clean_target or {}), + input=payload, + sensitive=sensitive, + before_state_id=before_id, + after_state_id=after_id, + before_observation_id=before_observation_id, + after_observation_id=after_observation_id, + observed_changes=observed, + proposed_postconditions=proposed, + status=status, + confidence=confidence, + evidence={ + **(evidence or {}), + "locator_match_count": match_count, + "locator_unique": unique, + "primary_locator": primary_expr, + }, + raw_event=dict(clean_raw_event or {}), + raw_event_ids=list(raw_event_ids or []), + ) + write_model_json(self.actions_dir / f"{act.id}.json", act) + self._actions_by_id[act.id] = act + self.action_flow.actions.append(act.id) + for observation_id in (before_observation_id, after_observation_id): + if observation_id and observation_id not in self.action_flow.observations: + self.action_flow.observations.append(observation_id) + if after_id: + self.action_flow.final_state_id = after_id + self.last_action = act + self._persist_flow() + self._notify_action_update(act, "captured") + if self.on_action is not None and action_type != "checkpoint": + try: + self.on_action(act) + except Exception as exc: + self.last_warning = f"action evidence scheduling failed: {exc}" + # Dual-write optional pointer fields into flow.json via side file is enough; + # artifacts integration updates flow.json separately. + return act + + def attach_eventual_evidence( + self, + action_ids: list[str], + *, + dump: Any, + requested_monotonic: float, + ) -> None: + """Attach one coalesced full dump without rewriting causal after-state.""" + with self._lock: + lag_ms = max(0, int((time.monotonic() - requested_monotonic) * 1000)) + state = self.snapshots.capture_from_dump( + dump, change_reason="eventual_after_actions", force_persist=True + ) + observation = self.snapshots.latest_observation + if state.id not in self.action_flow.states: + self.action_flow.states.append(state.id) + if observation and observation.id not in self.action_flow.observations: + self.action_flow.observations.append(observation.id) + self.action_flow.final_state_id = state.id + if (self.action_flow.surface or "") == "windows": + self._promote_windows_targets_from_dump( + dump, action_ids=action_ids, evidence_lag_ms=lag_ms + ) + for action_id in action_ids: + action = self._actions_by_id.get(action_id) + if action is None: + continue + action.evidence.update( + { + "eventual_observation_id": observation.id if observation else "", + "eventual_state_id": state.id, + "evidence_association": "coalesced_eventual", + "evidence_lag_ms": lag_ms, + } + ) + write_model_json(self.actions_dir / f"{action.id}.json", action) + self._notify_action_update(action, "evidence") + self._persist_flow() + + @staticmethod + def _is_oversized_windows_target(target: dict[str, Any]) -> bool: + bounds = dict(target.get("bounds") or target.get("rect") or {}) + area = int(bounds.get("width") or 0) * int(bounds.get("height") or 0) + return str(target.get("control_type") or "") == "WindowControl" or area > 1_000_000 + + @staticmethod + def _is_broad_windows_target(target: dict[str, Any]) -> bool: + if ActionRecordingManager._is_oversized_windows_target(target): + return True + bounds = dict(target.get("bounds") or target.get("rect") or {}) + area = int(bounds.get("width") or 0) * int(bounds.get("height") or 0) + control_type = str(target.get("control_type") or "") + name = str(target.get("name") or target.get("automation_id") or "").lower() + layout_tokens = ( + "search_result", "listview", "list_view", "watermark", + "container", "layout", "clientbg", "panel", "pane", + ) + return ( + area > 250_000 + or ( + control_type in {"TreeControl", "PaneControl", "GroupControl"} + and any(token in name for token in layout_tokens) + ) + ) + + @staticmethod + def _unique_windows_row_candidate( + rows: list[dict[str, Any]], + ) -> Optional[dict[str, Any]]: + """Return a single visible result/list row when geometry is unreliable.""" + row_tokens = ( + "user_item", "result_item", "search_item", "contact_item", + "list_item", "listitem", "result_row", "contact_row", + ) + candidates: dict[tuple[Any, ...], tuple[int, dict[str, Any]]] = {} + for index, row in enumerate(rows): + if row.get("is_offscreen") is True: + continue + control_type = str(row.get("control_type") or "") + name = " ".join( + str(row.get(key) or "") + for key in ("name", "automation_id", "class_name") + ).lower() + if not ( + any(token in name for token in row_tokens) + or control_type in {"ListItemControl", "DataItemControl"} + ): + continue + bounds = dict(row.get("bounds") or row.get("rect") or {}) + width = int(bounds.get("width") or 0) + height = int(bounds.get("height") or 0) + if width < 8 or height < 8: + continue + key = ( + int(bounds.get("left") or 0), int(bounds.get("top") or 0), + int(bounds.get("right") or 0), int(bounds.get("bottom") or 0), + str(row.get("name") or ""), str(row.get("value") or ""), + ) + candidates[key] = (index, dict(row)) + if len(candidates) != 1: + return None + index, chosen = next(iter(candidates.values())) + chosen["bounds"] = dict(chosen.get("bounds") or chosen.get("rect") or {}) + chosen.pop("rect", None) + # A row container often has no accessible value of its own. Prefer an + # explicit row-name descendant over structural labels such as + # search_avatar_label, and strip WeCom rich-text markup. + row_depth = int(chosen.get("depth") or 0) + structural = { + "layout", "container", "controlui", "horizontal", "vertical", + "user_item", "list_item", "result_item", + } + descendants: list[tuple[int, str, str, dict[str, Any]]] = [] + preferred_tokens = ( + "search_name", "groupbuddyname", "contact_name", "display_name", + "result_name", "title_label", + ) + rejected_tokens = ("avatar", "layout", "container", "controlui") + for child in rows[index + 1 :]: + child_depth = int(child.get("depth") or 0) + if child_depth <= row_depth: + break + raw_value = str(child.get("value") or "").strip() + name = str(child.get("name") or "").strip() + clean_value = html.unescape(re.sub(r"<[^>]+>", "", raw_value)).strip() + lowered = name.lower() + semantic = clean_value or ( + name if name and not any(token in lowered for token in structural) else "" + ) + if not semantic or semantic == "[REDACTED]" or any( + token in lowered for token in rejected_tokens + ): + continue + rank = 0 if any(token in lowered for token in preferred_tokens) else 1 + descendants.append((rank, semantic, name, child)) + if descendants: + _rank, semantic, name, child = min(descendants, key=lambda item: item[0]) + chosen["semantic_name"] = semantic + chosen["row_descendant"] = { + "name": name, + "value": str(child.get("value") or "").strip(), + "control_type": child.get("control_type") or "", + } + if not str(chosen.get("value") or ""): + chosen["value"] = semantic + return chosen + + @staticmethod + def _windows_hit_candidate( + rows: list[dict[str, Any]], x: int, y: int + ) -> Optional[dict[str, Any]]: + interactive = { + "ButtonControl", "EditControl", "TreeItemControl", "ListItemControl", + "MenuItemControl", "CheckBoxControl", "RadioButtonControl", + "ComboBoxControl", "TabItemControl", "HyperlinkControl", + } + semantic_tokens = ( + "search", "searchedit", "edit", "input", "message", "send", + "comment", "reply", "搜索", "输入", "发送", "消息", + ) + candidates: list[tuple[tuple[int, int, int, int], dict[str, Any]]] = [] + for row in rows: + bounds = dict(row.get("bounds") or row.get("rect") or {}) + try: + inside = ( + int(bounds.get("left")) <= x <= int(bounds.get("right")) + and int(bounds.get("top")) <= y <= int(bounds.get("bottom")) + ) + except (TypeError, ValueError): + continue + if not inside or row.get("is_offscreen") is True: + continue + width = max(0, int(bounds.get("width") or 0)) + height = max(0, int(bounds.get("height") or 0)) + area = width * height + control_type = str(row.get("control_type") or "") + # Hairlines, separators and layout-only artifacts are never useful + # click targets even when they happen to be the smallest hit. + if ( + not area + or width < 4 + or height < 4 + or control_type == "WindowControl" + ): + continue + text = " ".join( + str(row.get(key) or "") + for key in ("name", "value", "automation_id", "class_name") + ).lower() + if any(token in text for token in ("separator", "seperator", "divider", "splitter")): + continue + # A named editor/result is stronger than a smaller generic list + # item covering the same point. + exact_editor = any( + token in text + for token in ("searchedit", "send_rich_edit", "compose_box", "search_box") + ) and "_layout" not in text + if exact_editor: + priority = 0 + elif any(token in text for token in semantic_tokens): + priority = 1 + elif control_type in interactive: + priority = 2 + else: + priority = 3 + layout_penalty = 1 if "layout" in text or "container" in text else 0 + redacted = 1 if "[redacted]" in text else 0 + candidates.append(((priority, layout_penalty, redacted, area), row)) + if not candidates: + return None + chosen = dict(min(candidates, key=lambda item: item[0])[1]) + bounds = dict(chosen.get("bounds") or chosen.get("rect") or {}) + chosen["bounds"] = bounds + chosen.pop("rect", None) + chosen["capture_point"] = {"x": x, "y": y} + return chosen + + @staticmethod + def _windows_candidate_quality(target: dict[str, Any]) -> tuple[int, int, int]: + bounds = dict(target.get("bounds") or target.get("rect") or {}) + area = int(bounds.get("width") or 0) * int(bounds.get("height") or 0) + control_type = str(target.get("control_type") or "") + text = " ".join( + str(target.get(key) or "") + for key in ("name", "value", "automation_id", "class_name") + ).lower() + interactive = control_type in { + "ButtonControl", "EditControl", "TreeItemControl", "ListItemControl", + "MenuItemControl", "CheckBoxControl", "RadioButtonControl", + "ComboBoxControl", "TabItemControl", "HyperlinkControl", + } + semantic = any( + token in text + for token in ("search", "edit", "input", "message", "send", "buddy", "result") + ) + exact_editor = any( + token in text + for token in ("searchedit", "send_rich_edit", "compose_box", "search_box") + ) and "_layout" not in text + return (2 if exact_editor else (1 if semantic else 0), 1 if interactive else 0, -area) + + @staticmethod + def _clean_windows_rich_text(value: str) -> str: + return html.unescape(re.sub(r"<[^>]+>", "", str(value or ""))).strip() + + @staticmethod + def _same_windows_target_scope( + first: dict[str, Any], second: dict[str, Any] + ) -> bool: + a = dict(first.get("window_scope") or {}) + b = dict(second.get("window_scope") or {}) + ah, bh = int(a.get("hwnd") or 0), int(b.get("hwnd") or 0) + if ah and bh: + return ah == bh + ap, bp = int(a.get("pid") or 0), int(b.get("pid") or 0) + ac = str(a.get("class_name") or first.get("class_name") or "") + bc = str(b.get("class_name") or second.get("class_name") or "") + return bool(ap and ap == bp and (not ac or not bc or ac == bc)) + + @staticmethod + def _windows_scope_key(target: dict[str, Any]) -> tuple[Any, ...]: + scope = dict(target.get("window_scope") or {}) + hwnd = int(scope.get("hwnd") or 0) + if hwnd: + return ("hwnd", hwnd) + return ( + "process", + int(scope.get("pid") or 0), + str(scope.get("class_name") or target.get("class_name") or ""), + ) + + @staticmethod + def _is_dynamic_windows_candidate(target: dict[str, Any]) -> bool: + name = str(target.get("name") or target.get("automation_id") or "").lower() + value = str(target.get("value") or "").strip() + dynamic_tokens = ( + "description", "preview", "snippet", "latest_message", + "last_message", "timestamp", "time_label", "消息摘要", "最新消息", + ) + return bool(value and any(token in name for token in dynamic_tokens)) + + @staticmethod + def _same_windows_scope(target: dict[str, Any], window: dict[str, Any]) -> bool: + scope = dict(target.get("window_scope") or {}) + target_hwnd = int(scope.get("hwnd") or 0) + dump_hwnd = int(window.get("hwnd") or 0) + if target_hwnd and dump_hwnd: + return target_hwnd == dump_hwnd + target_pid = int(scope.get("pid") or 0) + dump_pid = int(window.get("pid") or 0) + target_class = str(scope.get("class_name") or target.get("class_name") or "") + dump_class = str(window.get("class_name") or "") + if not scope: + # Synthetic/unit captures may only carry the root class. Real + # Windows events include hwnd/pid; class equality is the safest + # compatibility fallback when that identity is unavailable. + return bool(target_class and target_class == dump_class) + return bool(target_pid and target_pid == dump_pid and target_class == dump_class) + + def _promote_windows_targets_from_dump( + self, + dump: Any, + *, + action_ids: Optional[list[str]] = None, + evidence_lag_ms: int = 0, + ) -> None: + rows = list(getattr(dump, "locatable", None) or []) + window = dict(getattr(dump, "window", None) or {}) + if not rows: + return + last_business_by_scope: dict[tuple[Any, ...], dict[str, Any]] = {} + allowed = set(action_ids or []) + actions = sorted(self._actions_by_id.values(), key=lambda item: item.sequence) + for action in actions: + target = dict(action.target_snapshot or {}) + scope_key = self._windows_scope_key(target) + should_update = not allowed or action.id in allowed + if not should_update: + if target and not self._is_broad_windows_target(target): + last_business_by_scope[scope_key] = target + continue + # Eventual evidence from another application or from WeCom's main + # window must never rewrite a transient popup/screenshot action. + if not self._same_windows_scope(target, window): + continue + promoted: Optional[dict[str, Any]] = None + promotion_reason = "eventual_tree_coordinate_hit" + if action.action_type == "click" and self._is_broad_windows_target(target): + raw = dict(action.raw_event or {}) + meta = dict(raw.get("meta") or {}) + point = dict(meta.get("uia_coordinates") or {}) + points: list[tuple[int, int]] = [] + x = point.get("x", raw.get("x")) + y = point.get("y", raw.get("y")) + if x is not None and y is not None: + points.append((int(x), int(y))) + for candidate in meta.get("uia_coordinate_candidates") or []: + cx, cy = candidate.get("x"), candidate.get("y") + if cx is not None and cy is not None: + p = (int(cx), int(cy)) + if p not in points: + points.append(p) + hits = [ + hit + for px, py in points + for hit in [self._windows_hit_candidate(rows, px, py)] + if hit is not None + ] + if hits: + promoted = max(hits, key=self._windows_candidate_quality) + window_class = str(window.get("class_name") or "") + if ( + (not promoted or self._is_broad_windows_target(promoted)) + and window_class == "SearchResultWindow2" + ): + unique_row = self._unique_windows_row_candidate(rows) + if unique_row: + promoted = unique_row + promotion_reason = "eventual_tree_unique_row" + elif action.action_type in {"fill", "type", "press_key"} and self._is_oversized_windows_target(target): + promoted = dict(last_business_by_scope.get(scope_key) or {}) or None + elif target and not self._is_broad_windows_target(target): + last_business_by_scope[scope_key] = target + + if not promoted: + continue + if evidence_lag_ms > 5_000 and self._is_dynamic_windows_candidate(promoted): + action.evidence.update( + { + "target_promotion_rejected": "late_dynamic_candidate", + "rejected_candidate_name": promoted.get("name") or "", + "rejected_evidence_lag_ms": int(evidence_lag_ms), + } + ) + write_model_json(self.actions_dir / f"{action.id}.json", action) + continue + promoted["window_scope"] = dict( + promoted.get("window_scope") or target.get("window_scope") or window + ) + promoted_name = str(promoted.get("name") or "").lower() + if any( + token in promoted_name + for token in ("search_name", "contact_name", "groupbuddyname", "display_name") + ): + cleaned_value = self._clean_windows_rich_text( + str(promoted.get("value") or "") + ) + if cleaned_value: + promoted["value"] = cleaned_value + promoted["semantic_name"] = cleaned_value + element = self.catalog.upsert_from_snapshot( + promoted, + surface="windows", + window_scope=promoted.get("window_scope") or {}, + rows=rows, + ) + action.target_snapshot = sanitize_artifact(promoted) + action.target_element_id = element.id + action.confidence = max(float(action.confidence or 0.0), 0.72) + action.evidence.update( + { + "target_promotion": promotion_reason, + "original_target_control_type": target.get("control_type") or "", + "locator_match_count": ( + element.primary_locator.actual_count if element.primary_locator else None + ), + "locator_unique": ( + element.primary_locator.unique if element.primary_locator else None + ), + "primary_locator": ( + element.primary_locator.expression if element.primary_locator else "" + ), + } + ) + if isinstance(action.raw_event, dict): + action.raw_event["target"] = sanitize_artifact(promoted) + if element.id not in self.action_flow.elements: + self.action_flow.elements.append(element.id) + last_business_by_scope[scope_key] = promoted + write_model_json(self.actions_dir / f"{action.id}.json", action) + + def note_evidence_failure(self, action_ids: list[str], error: str) -> None: + with self._lock: + self.last_warning = f"eventual evidence failed: {error}" + for action_id in action_ids: + action = self._actions_by_id.get(action_id) + if action is None: + continue + action.evidence["eventual_evidence_error"] = str(error) + write_model_json(self.actions_dir / f"{action.id}.json", action) + self._notify_action_update(action, "evidence_failed") + + def _notify_action_update(self, action: ActionRecord, phase: str) -> None: + if self.on_action_update is None: + return + try: + self.on_action_update(action, phase) + except Exception as exc: + self.last_warning = f"action UI update failed: {exc}" + + def _persist_flow(self) -> None: + write_model_json(self.flow_dir / "action_flow.json", self.action_flow) + + def heartbeat(self) -> None: + """Lightweight monitor: flush idle input; do NOT create states/actions.""" + if self.status != RecorderState.RECORDING: + return + with self._lock: + for item in self.aggregator.maybe_flush_idle(): + if item.action_type == "press_key": + self._commit_press_key(item) + else: + self._commit_fill(item) + + def stats(self) -> dict[str, Any]: + la = self.last_action + return { + "recorder_status": self.status, + "state_count": len(self.action_flow.states), + "observation_count": len(self.snapshots.observation_ids()), + "action_count": len(self.action_flow.actions), + "element_count": len(self.catalog.catalog.elements), + "last_action_type": la.action_type if la else "", + "last_target": (la.target_snapshot or {}).get("name") + or (la.target_snapshot or {}).get("automation_id") + or (la.target_snapshot or {}).get("semantic_name") + or (la.target_snapshot or {}).get("class_name") + or (la.target_snapshot or {}).get("tag") + if la + else "", + "last_match_count": (la.evidence or {}).get("locator_match_count") if la else None, + "last_unique": (la.evidence or {}).get("locator_unique") if la else None, + "last_page_id": ((la.raw_event or {}).get("meta") or {}).get("page_id") if la else "", + "last_observation_id": ( + la.after_observation_id + or (la.evidence or {}).get("eventual_observation_id") + if la else "" + ), + "warning": self.last_warning, + "recording_mode": self.action_flow.recording_mode, + } diff --git a/tools/ui_inspector/recorder/windows.py b/tools/ui_inspector/recorder/windows.py new file mode 100644 index 0000000..de1644d --- /dev/null +++ b/tools/ui_inspector/recorder/windows.py @@ -0,0 +1,1711 @@ +"""Event-driven Windows action recorder. + +Input is observed out-of-process with low-level Win32 hooks. Hook callbacks do +only constant-time queue writes; UIA resolution happens on a separate worker so +a slow provider (WeCom/SAP/etc.) cannot make Windows remove the hook. The old +``GetAsyncKeyState`` polling path remains only as a mouse fallback when hook +installation is unavailable. +""" + +from __future__ import annotations + +import ctypes +import os +import queue +import threading +import time +from ctypes import wintypes +from typing import Any, Callable, Optional + +from .base import RawUserEvent, RecorderBase, RecorderState + +# Win32 virtual key codes +_VK_LBUTTON = 0x01 +_VK_RBUTTON = 0x02 +_VK_MBUTTON = 0x04 + +_WH_KEYBOARD_LL = 13 +_WH_MOUSE_LL = 14 +_WM_QUIT = 0x0012 +_WM_KEYDOWN = 0x0100 +_WM_SYSKEYDOWN = 0x0104 +_WM_LBUTTONDOWN = 0x0201 +_WM_LBUTTONUP = 0x0202 +_WM_RBUTTONDOWN = 0x0204 +_WM_RBUTTONUP = 0x0205 +_LLMHF_INJECTED = 0x00000001 + + +class _POINT(ctypes.Structure): + _fields_ = [("x", wintypes.LONG), ("y", wintypes.LONG)] + + +class _MSLLHOOKSTRUCT(ctypes.Structure): + _fields_ = [ + ("pt", _POINT), + ("mouseData", wintypes.DWORD), + ("flags", wintypes.DWORD), + ("time", wintypes.DWORD), + ("dwExtraInfo", ctypes.c_size_t), + ] + + +class _KBDLLHOOKSTRUCT(ctypes.Structure): + _fields_ = [ + ("vkCode", wintypes.DWORD), + ("scanCode", wintypes.DWORD), + ("flags", wintypes.DWORD), + ("time", wintypes.DWORD), + ("dwExtraInfo", ctypes.c_size_t), + ] + + +_SPECIAL_KEYS = { + 0x08: "Backspace", + 0x09: "Tab", + 0x0D: "Enter", + 0x1B: "Escape", + 0x21: "PageUp", + 0x22: "PageDown", + 0x23: "End", + 0x24: "Home", + 0x25: "ArrowLeft", + 0x26: "ArrowUp", + 0x27: "ArrowRight", + 0x28: "ArrowDown", + 0x2E: "Delete", + **{0x70 + i: f"F{i + 1}" for i in range(12)}, +} + + +class WindowsRecorder(RecorderBase): + """Observe Windows input and resolve UIA targets outside target processes.""" + + capability_map = { + "click": "implemented", + "fill": "implemented", + "press_key": "implemented", + "double_click": "schema_only", + "right_click": "partial", + } + + def __init__( + self, + *, + on_event: Optional[Callable[[RawUserEvent], None]] = None, + exclude_hwnds: Optional[Callable[[], set[int]]] = None, + poll_sec: float = 0.03, + focus_value_poll_sec: float = 0.4, + ) -> None: + super().__init__(on_event=on_event, exclude_hwnds=exclude_hwnds) + self.poll_sec = poll_sec + self.focus_value_poll_sec = focus_value_poll_sec + self._prev_down = False + self._last_focus_value = "" + self._last_focus_fp = "" + self._pending_target: dict[str, Any] = {} + self._last_uia_point: Optional[tuple[int, int]] = None + self._last_raw_point: Optional[tuple[int, int]] = None + self._last_root_hwnd = 0 + self._last_actionable_target: dict[str, Any] = {} + # Text editors need their own sticky target. A generic "last + # actionable" target is unsafe here: selecting a contact immediately + # before typing would otherwise make the contact row the input target. + self._last_text_entry_target: dict[str, Any] = {} + self._live_text_control: Any = None + self._element_atlas: dict[str, list[dict[str, Any]]] = {} + self._atlas_lock = threading.RLock() + self._ime_composing = False + self._last_typing_activity_identity = "" + self._last_typing_activity_at = 0.0 + self._input_queue: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=1024) + self._capture_thread: Optional[threading.Thread] = None + self._uia_thread: Optional[threading.Thread] = None + self._uia_request = threading.Event() + self._uia_finished = threading.Event() + self._snapshot_lock = threading.RLock() + self._latest_focus_snapshot: dict[str, Any] = {} + self._latest_focus_sampled_at = 0.0 + self._text_target_generation = 0 + self._uia_bound_target_generation = -1 + self._typing_active = False + self._last_key_at = 0.0 + self._accept_input = True + self._startup_ready = threading.Event() + self._hook_thread_id = 0 + self._mouse_hook = None + self._keyboard_hook = None + self._mouse_proc = None + self._keyboard_proc = None + self._health_lock = threading.Lock() + self._health: dict[str, Any] = { + "backend": "starting", + "mouse_hook": False, + "keyboard_hook": False, + "raw_input_events": 0, + "mouse_events": 0, + "keyboard_events": 0, + "resolved_events": 0, + "queue_dropped": 0, + "loop_exceptions": 0, + "last_error": "", + "last_event_monotonic": None, + "undrained_events": 0, + "shutdown_complete": True, + "uia_samples": 0, + } + + @property + def capability(self) -> dict[str, str]: + return dict(self.capability_map) + + def start(self) -> None: + if self._thread and self._thread.is_alive(): + self.status = RecorderState.RECORDING + return + self._stop.clear() + self._startup_ready.clear() + self._uia_finished.clear() + self._uia_request.clear() + self._accept_input = True + self._health_set(shutdown_complete=False, undrained_events=0) + self.status = RecorderState.RECORDING + self._capture_thread = threading.Thread( + target=self._capture_loop, + name="ui-inspector-win-input-consumer", + daemon=True, + ) + self._capture_thread.start() + self._uia_thread = threading.Thread( + target=self._uia_sampler_loop, + name="ui-inspector-win-uia-sampler", + daemon=True, + ) + self._uia_thread.start() + self._thread = threading.Thread( + target=self._loop, name="ui-inspector-win-recorder", daemon=True + ) + self._thread.start() + if not self._startup_ready.wait(timeout=2.0): + self._record_error("Windows input backend did not complete startup handshake") + + def stop(self) -> None: + # Stop accepting hook packets first. The consumer then drains every + # packet already accepted while the independent UIA sampler performs + # one final read. Never report a clean shutdown with a hidden tail. + self._accept_input = False + self._stop.set() + self._uia_request.set() + if self._hook_thread_id: + try: + ctypes.windll.user32.PostThreadMessageW( + int(self._hook_thread_id), _WM_QUIT, 0, 0 + ) + except Exception as exc: + self._record_error(exc) + if self._thread and self._thread.is_alive(): + self._thread.join(timeout=2.0) + listener_alive = bool(self._thread and self._thread.is_alive()) + if not listener_alive: + self._thread = None + if self._uia_thread and self._uia_thread.is_alive(): + self._uia_thread.join(timeout=3.0) + uia_alive = bool(self._uia_thread and self._uia_thread.is_alive()) + if not uia_alive: + self._uia_thread = None + if self._capture_thread and self._capture_thread.is_alive(): + self._capture_thread.join(timeout=5.0) + resolver_alive = bool(self._capture_thread and self._capture_thread.is_alive()) + if not resolver_alive: + self._capture_thread = None + undrained = int(self._input_queue.qsize()) + complete = not listener_alive and not resolver_alive and undrained == 0 + self._health_set( + undrained_events=undrained, + shutdown_complete=complete, + uia_sampler_still_running=uia_alive, + ) + self.status = RecorderState.STOPPED + if not complete: + raise RuntimeError( + "desktop recorder did not shut down cleanly: " + f"undrained={undrained}, listener_alive={listener_alive}, " + f"consumer_alive={resolver_alive}" + ) + + def health(self) -> dict[str, Any]: + with self._health_lock: + result = dict(self._health) + result["listener_thread_alive"] = bool( + self._thread and self._thread.is_alive() + ) + result["resolver_thread_alive"] = bool( + self._capture_thread and self._capture_thread.is_alive() + ) + result["uia_thread_alive"] = bool( + self._uia_thread and self._uia_thread.is_alive() + ) + result["input_queue_size"] = self._input_queue.qsize() + return result + + def _health_add(self, key: str, amount: int = 1) -> None: + with self._health_lock: + self._health[key] = int(self._health.get(key) or 0) + amount + + def _health_set(self, **values: Any) -> None: + with self._health_lock: + self._health.update(values) + + def _record_error(self, exc: Any) -> None: + self._health_add("loop_exceptions") + self._health_set(last_error=f"{type(exc).__name__}: {exc}" if isinstance(exc, BaseException) else str(exc)) + + def _loop(self) -> None: + if os.name != "nt": + self._health_set(backend="unsupported") + self._startup_ready.set() + return + try: + if self._run_hook_message_loop(): + return + except Exception as exc: + self._record_error(exc) + # Compatibility path: never the preferred backend. + self._health_set(backend="get_async_key_state_fallback") + self._startup_ready.set() + self._poll_mouse_fallback() + + def _run_hook_message_loop(self) -> bool: + user32 = ctypes.windll.user32 + kernel32 = ctypes.windll.kernel32 + lresult = ctypes.c_ssize_t + hook_proc_type = ctypes.WINFUNCTYPE( + lresult, ctypes.c_int, ctypes.c_size_t, ctypes.c_ssize_t + ) + user32.SetWindowsHookExW.argtypes = [ + ctypes.c_int, hook_proc_type, ctypes.c_void_p, wintypes.DWORD + ] + user32.SetWindowsHookExW.restype = ctypes.c_void_p + user32.CallNextHookEx.argtypes = [ + ctypes.c_void_p, ctypes.c_int, ctypes.c_size_t, ctypes.c_ssize_t + ] + user32.CallNextHookEx.restype = lresult + user32.UnhookWindowsHookEx.argtypes = [ctypes.c_void_p] + user32.UnhookWindowsHookEx.restype = wintypes.BOOL + kernel32.GetModuleHandleW.restype = ctypes.c_void_p + + @hook_proc_type + def mouse_proc(code, wparam, lparam): + if code >= 0 and wparam in { + _WM_LBUTTONDOWN, _WM_LBUTTONUP, _WM_RBUTTONDOWN, _WM_RBUTTONUP + }: + data = ctypes.cast( + lparam, ctypes.POINTER(_MSLLHOOKSTRUCT) + ).contents + self._enqueue_input( + { + "kind": "mouse", + "message": int(wparam), + "x": int(data.pt.x), + "y": int(data.pt.y), + "injected": bool(data.flags & _LLMHF_INJECTED), + "source_time_ms": int(data.time), + "coordinate_space": "physical_screen", + } + ) + return user32.CallNextHookEx(None, code, wparam, lparam) + + @hook_proc_type + def keyboard_proc(code, wparam, lparam): + if code >= 0 and wparam in {_WM_KEYDOWN, _WM_SYSKEYDOWN}: + data = ctypes.cast( + lparam, ctypes.POINTER(_KBDLLHOOKSTRUCT) + ).contents + self._enqueue_input( + { + "kind": "key", + "vk": int(data.vkCode), + "scan": int(data.scanCode), + "injected": bool(data.flags & 0x10), + "source_time_ms": int(data.time), + } + ) + return user32.CallNextHookEx(None, code, wparam, lparam) + + # Keep callbacks alive for the complete lifetime of the hooks. + self._mouse_proc = mouse_proc + self._keyboard_proc = keyboard_proc + module = kernel32.GetModuleHandleW(None) + self._hook_thread_id = int(kernel32.GetCurrentThreadId()) + self._mouse_hook = user32.SetWindowsHookExW( + _WH_MOUSE_LL, mouse_proc, module, 0 + ) + self._keyboard_hook = user32.SetWindowsHookExW( + _WH_KEYBOARD_LL, keyboard_proc, module, 0 + ) + mouse_ok = bool(self._mouse_hook) + keyboard_ok = bool(self._keyboard_hook) + self._health_set( + backend="low_level_hooks" if mouse_ok else "hook_failed", + mouse_hook=mouse_ok, + keyboard_hook=keyboard_ok, + ) + self._startup_ready.set() + if not mouse_ok: + if keyboard_ok: + user32.UnhookWindowsHookEx(self._keyboard_hook) + self._keyboard_hook = None + return False + + msg = wintypes.MSG() + try: + while not self._stop.is_set(): + result = int(user32.GetMessageW(ctypes.byref(msg), None, 0, 0)) + if result <= 0: + if result < 0: + self._record_error("GetMessageW failed") + break + user32.TranslateMessage(ctypes.byref(msg)) + user32.DispatchMessageW(ctypes.byref(msg)) + finally: + if self._mouse_hook: + user32.UnhookWindowsHookEx(self._mouse_hook) + if self._keyboard_hook: + user32.UnhookWindowsHookEx(self._keyboard_hook) + self._mouse_hook = None + self._keyboard_hook = None + self._hook_thread_id = 0 + return True + + def _poll_mouse_fallback(self) -> None: + user32 = ctypes.windll.user32 + while not self._stop.is_set(): + if self.status != RecorderState.RECORDING: + time.sleep(0.05) + continue + try: + down = bool(user32.GetAsyncKeyState(_VK_LBUTTON) & 0x8000) + if down and not self._prev_down: + x, y = self._cursor_pos(user32) + self._enqueue_input( + {"kind": "mouse", "message": _WM_LBUTTONDOWN, "x": x, "y": y, "fallback": True, "coordinate_space": "uia_screen"} + ) + elif not down and self._prev_down: + self._enqueue_input({"kind": "mouse", "message": _WM_LBUTTONUP, "fallback": True}) + self._prev_down = down + except Exception as exc: + self._record_error(exc) + time.sleep(self.poll_sec) + + def _enqueue_input(self, packet: dict[str, Any]) -> None: + if not self._accept_input: + self._health_add("input_rejected_after_stop") + return + self._health_add("raw_input_events") + self._health_add("mouse_events" if packet.get("kind") == "mouse" else "keyboard_events") + self._health_set(last_event_monotonic=time.monotonic()) + try: + self._input_queue.put_nowait(packet) + except queue.Full: + self._health_add("queue_dropped") + + def _capture_loop(self) -> None: + # This is deliberately a UIA-free path. Its only job is to preserve + # input order and turn cached snapshots into semantic events quickly. + while ( + not self._stop.is_set() + or not self._input_queue.empty() + or not self._uia_finished.is_set() + ): + packet = None + try: + packet = self._input_queue.get(timeout=0.05) + except queue.Empty: + pass + try: + if packet is not None and self.status == RecorderState.RECORDING: + self._dispatch_input_packet(packet) + self._health_add("resolved_events") + if ( + self._typing_active + and time.monotonic() - self._last_key_at >= 0.25 + ): + self._emit_cached_value_change(force=False) + except Exception as exc: + self._record_error(exc) + finally: + if packet is not None: + self._input_queue.task_done() + self._emit_cached_value_change(force=True) + + def _uia_sampler_loop(self) -> None: + """Own all UIA/COM objects and publish plain-dict snapshots only.""" + try: + import desktop_uia as uia + uia.ensure_com_sta() + except Exception as exc: + self._record_error(exc) + try: + while True: + self._uia_request.wait(timeout=self.focus_value_poll_sec) + self._uia_request.clear() + try: + with self._snapshot_lock: + generation = self._text_target_generation + requested_target = dict(self._last_text_entry_target or {}) + if ( + requested_target + and generation != self._uia_bound_target_generation + ): + self._bind_text_target(requested_target, resolve_live=True) + self._uia_bound_target_generation = generation + # Once an editor has been bound, read it directly even if a + # later click has moved OS focus. This preserves the final + # committed value at transaction boundaries. + live_info = self._snapshot_live_text_control() + info = self._focused_control_snapshot() + if live_info and self._typing_active: + info = live_info + if info: + self._publish_focus_snapshot(info) + self._health_add("uia_samples") + except Exception as exc: + self._record_error(exc) + if self._stop.is_set(): + break + finally: + self._uia_finished.set() + + def _publish_focus_snapshot(self, info: dict[str, Any]) -> None: + """Cross the UIA thread boundary using data, never COM controls.""" + if not info or self._is_transient_input_target(info): + return + snap = dict(info) + with self._snapshot_lock: + self._latest_focus_snapshot = snap + self._latest_focus_sampled_at = time.monotonic() + if self._is_text_entry(snap): + self._last_text_entry_target = dict(snap) + + def _cached_focus_snapshot(self) -> dict[str, Any]: + with self._snapshot_lock: + return dict(self._latest_focus_snapshot or {}) + + @staticmethod + def _focus_fingerprint(info: dict[str, Any]) -> str: + scope = dict(info.get("window_scope") or {}) + return "|".join( + [ + str(scope.get("hwnd") or 0), + str(info.get("automation_id") or ""), + str(info.get("name") or ""), + str(info.get("control_type") or ""), + ] + ) + + def _emit_cached_value_change(self, *, force: bool) -> None: + info = self._cached_focus_snapshot() + if not info or not self._is_text_entry(info): + return + fp = self._focus_fingerprint(info) + value = str(info.get("value") or "") + if fp != self._last_focus_fp: + # A late first sample may already contain the completed text. If + # keys were observed, preserve it instead of mistaking it for the + # baseline of a newly focused editor. + if not (force and self._typing_active and value): + self._last_focus_fp = fp + self._last_focus_value = value + return + if value == self._last_focus_value: + return + self.emit( + RawUserEvent( + kind="value_change", + surface="windows", + text=value, + target=info, + meta={ + "input_type": "password" + if "password" in str(info.get("name") or "").lower() + else "", + "sampled_at": self._latest_focus_sampled_at, + "causal_flush": bool(force), + }, + ) + ) + self._last_focus_fp = fp + self._last_focus_value = value + self._typing_active = False + + def _dispatch_input_packet(self, packet: dict[str, Any]) -> None: + message = int(packet.get("message") or 0) + if packet.get("kind") == "mouse": + if message in {_WM_LBUTTONDOWN, _WM_RBUTTONDOWN}: + self._on_mouse_down_at_fast( + int(packet.get("x") or 0), + int(packet.get("y") or 0), + button="right" if message == _WM_RBUTTONDOWN else "left", + source_meta=packet, + ) + elif message in {_WM_LBUTTONUP, _WM_RBUTTONUP}: + self._on_mouse_up() + elif packet.get("kind") == "key": + self._on_key_down(packet) + + def _on_mouse_down(self, user32) -> None: + point = self._cursor_pos(user32) + self._on_mouse_down_at_fast(*point) + + def _on_mouse_down_at_fast( + self, + x: int, + y: int, + *, + button: str = "left", + source_meta: Optional[dict[str, Any]] = None, + ) -> None: + """Resolve a click without touching UIA. + + Win32 supplies application ownership and the persisted element atlas + supplies an immediate semantic target. The sampler is then nudged to + refine focus/value state asynchronously. + """ + self._emit_cached_value_change(force=True) + raw_x, raw_y = int(x), int(y) + user32 = ctypes.windll.user32 + exclude: set[int] = set() + try: + exclude = set(self.exclude_hwnds() or set()) + except Exception: + pass + hwnd = 0 + root = 0 + pid = 0 + try: + point = self._make_point(raw_x, raw_y) + physical_lookup = getattr(user32, "WindowFromPhysicalPoint", None) + if physical_lookup: + physical_lookup.restype = ctypes.c_void_p + hwnd = int(physical_lookup(point) or 0) + else: + user32.WindowFromPoint.restype = ctypes.c_void_p + hwnd = int(user32.WindowFromPoint(point) or 0) + root = int(user32.GetAncestor(hwnd, 2) or hwnd) if hwnd else 0 + pid_ref = ctypes.c_ulong() + if root: + user32.GetWindowThreadProcessId(root, ctypes.byref(pid_ref)) + pid = int(pid_ref.value or 0) + if root in exclude or hwnd in exclude or pid == os.getpid(): + return + overlay = self._top_level_window_at_point( + raw_x, raw_y, exclude_hwnds=exclude, preferred_pid=pid + ) + if overlay and overlay != root: + root = int(overlay) + pid_ref = ctypes.c_ulong() + user32.GetWindowThreadProcessId(root, ctypes.byref(pid_ref)) + pid = int(pid_ref.value or 0) + except Exception as exc: + self._record_error(exc) + + scope = self._window_scope_fast(root or hwnd, fallback_pid=pid) + points = [(raw_x, raw_y)] + scale = 1.0 + try: + get_dpi = getattr(user32, "GetDpiForWindow", None) + dpi = int(get_dpi(int(root or hwnd)) or 96) if get_dpi and (root or hwnd) else 96 + scale = max(1.0, dpi / 96.0) + if scale > 1.05: + points.append((round(raw_x / scale), round(raw_y / scale))) + except Exception: + pass + target = self._atlas_hit(str(scope.get("class_name") or ""), points) + if target: + target["window_scope"] = dict(scope) + selection = "persisted_element_atlas" + else: + target = { + "name": str(scope.get("title") or ""), + "control_type": "WindowControl", + "class_name": str(scope.get("class_name") or ""), + "window_scope": dict(scope), + } + selection = "win32_window_fallback" + + self._last_raw_point = (raw_x, raw_y) + # Atlas coordinates are commonly logical on mixed-DPI desktop apps. + self._last_uia_point = points[-1] + self._last_root_hwnd = int(root or hwnd or 0) + self._pending_target = dict(target) + if not self._is_root_or_oversized(target): + self._last_actionable_target = dict(target) + if self._is_text_entry(target): + self._set_text_target_fast(target) + self.emit( + RawUserEvent( + kind="click", + surface="windows", + button=button, + x=raw_x, + y=raw_y, + target=target, + meta={ + "input_backend": self.health().get("backend"), + "injected": bool((source_meta or {}).get("injected")), + "source_time_ms": (source_meta or {}).get("source_time_ms"), + "source_coordinate_space": str( + (source_meta or {}).get("coordinate_space") or "uia_screen" + ), + "raw_coordinates_physical": {"x": raw_x, "y": raw_y}, + "coordinate_selection": selection, + "dpi_scale": scale, + "uia_deferred": True, + }, + ) + ) + self._uia_request.set() + + @staticmethod + def _window_scope_fast(hwnd: int, *, fallback_pid: int = 0) -> dict[str, Any]: + scope = {"hwnd": int(hwnd or 0), "pid": int(fallback_pid or 0), "title": "", "class_name": ""} + if not hwnd: + return scope + try: + user32 = ctypes.windll.user32 + title = ctypes.create_unicode_buffer(512) + klass = ctypes.create_unicode_buffer(256) + user32.GetWindowTextW(int(hwnd), title, len(title)) + user32.GetClassNameW(int(hwnd), klass, len(klass)) + pid_ref = ctypes.c_ulong() + user32.GetWindowThreadProcessId(int(hwnd), ctypes.byref(pid_ref)) + scope.update( + title=title.value, + class_name=klass.value, + pid=int(pid_ref.value or fallback_pid or 0), + ) + except Exception: + pass + return scope + + def _set_text_target_fast(self, info: dict[str, Any]) -> None: + with self._snapshot_lock: + self._last_text_entry_target = dict(info) + self._text_target_generation += 1 + self._last_focus_fp = self._focus_fingerprint(info) + self._last_focus_value = str(info.get("value") or "") + with self._snapshot_lock: + # The atlas target is useful immediately. The sampler will replace + # it with a live value-bearing snapshot when the provider responds. + self._latest_focus_snapshot = dict(info) + self._latest_focus_sampled_at = time.monotonic() + self._typing_active = False + self._uia_request.set() + + def _on_mouse_down_at( + self, + x: int, + y: int, + *, + button: str = "left", + source_meta: Optional[dict[str, Any]] = None, + ) -> None: + user32 = ctypes.windll.user32 + # Preserve the final value of the previously focused editor before the + # click changes focus (important for paste-then-click workflows). + if self._last_focus_fp: + self._poll_focus_value() + raw_x, raw_y = int(x), int(y) + exclude = set() + try: + exclude = set(self.exclude_hwnds() or set()) + except Exception: + exclude = set() + + # Filter Inspector window + root = 0 + try: + point = self._make_point(raw_x, raw_y) + physical_lookup = getattr(user32, "WindowFromPhysicalPoint", None) + user32.WindowFromPoint.restype = ctypes.c_void_p + if physical_lookup: + physical_lookup.restype = ctypes.c_void_p + hwnd = int( + (physical_lookup(point) if physical_lookup else user32.WindowFromPoint(point)) + or 0 + ) + root = int(user32.GetAncestor(hwnd, 2) or hwnd) if hwnd else 0 # GA_ROOT=2 + target_pid = ctypes.c_ulong() + if root: + user32.GetWindowThreadProcessId(root, ctypes.byref(target_pid)) + if ( + root in exclude + or hwnd in exclude + or int(target_pid.value or 0) == os.getpid() + ): + self.emit( + RawUserEvent( + kind="mouse_down", + surface="windows", + x=x, + y=y, + from_inspector=True, + meta={"reason": "inspector_hwnd"}, + ) + ) + return + overlay = self._top_level_window_at_point( + raw_x, + raw_y, + exclude_hwnds=exclude, + preferred_pid=int(target_pid.value or 0), + ) + if overlay and overlay != root: + root = overlay + target_pid = ctypes.c_ulong() + user32.GetWindowThreadProcessId(root, ctypes.byref(target_pid)) + except Exception: + hwnd = 0 + + uia_x, uia_y, dpi_meta = self._uia_point_for_window( + root or hwnd, raw_x, raw_y, + coordinate_space=str((source_meta or {}).get("coordinate_space") or "uia_screen"), + ) + target, err = self._element_from_point(uia_x, uia_y) + target_scope_hwnd = int((target.get("window_scope") or {}).get("hwnd") or 0) + target_scope_root = ( + int(user32.GetAncestor(target_scope_hwnd, 2) or target_scope_hwnd) + if target_scope_hwnd + else 0 + ) + if root and target_scope_root and target_scope_root != int(root): + root_target, root_error = self._element_from_handle(int(root)) + if root_target: + target = root_target + err = root_error + target, uia_x, uia_y, probe_meta = self._probe_mixed_uia_coordinates( + root or hwnd, + raw_x, + raw_y, + current_point=(uia_x, uia_y), + current_target=target or {}, + ) + dpi_meta.update(probe_meta) + target_scope = dict((target or {}).get("window_scope") or {}) + atlas_class = str(target_scope.get("class_name") or "") + if self._is_root_or_oversized(target or {}) and atlas_class: + atlas_points = [(int(uia_x), int(uia_y)), (raw_x, raw_y)] + for candidate in probe_meta.get("uia_coordinate_candidates") or []: + if candidate.get("x") is not None and candidate.get("y") is not None: + atlas_points.append((int(candidate["x"]), int(candidate["y"]))) + atlas_target = self._atlas_hit(atlas_class, atlas_points) + if atlas_target: + atlas_target["window_scope"] = dict(target_scope) + target = atlas_target + dpi_meta["coordinate_selection"] = "persisted_element_atlas" + dpi_meta["uia_coordinates"] = {"x": uia_x, "y": uia_y} + self._last_uia_point = (uia_x, uia_y) + self._last_raw_point = (raw_x, raw_y) + self._last_root_hwnd = int(root or hwnd or 0) + self._pending_target = target or {} + if self._pending_target and not self._is_root_or_oversized(self._pending_target): + self._last_actionable_target = dict(self._pending_target) + if self._is_text_entry(self._pending_target): + self._bind_text_target(self._pending_target) + self._seed_focus_baseline(self._pending_target) + meta: dict[str, Any] = { + "input_backend": self.health().get("backend"), + "injected": bool((source_meta or {}).get("injected")), + "source_time_ms": (source_meta or {}).get("source_time_ms"), + **dpi_meta, + } + if err: + meta["element_from_point_error"] = err + meta["raw_coordinates"] = {"x": x, "y": y} + # Coordinates saved as evidence only — never auto primary locator. + meta["coordinates_are_not_primary_locator"] = True + self.emit( + RawUserEvent( + kind="click", + surface="windows", + button=button, + x=uia_x, + y=uia_y, + target=self._pending_target, + meta=meta, + ) + ) + + @staticmethod + def _uia_point_for_window( + hwnd: int, + x: int, + y: int, + *, + coordinate_space: str, + ) -> tuple[int, int, dict[str, Any]]: + """Map WH_*_LL physical coordinates into the DPI-virtualized UIA space.""" + meta: dict[str, Any] = { + "raw_coordinates_physical": {"x": int(x), "y": int(y)}, + "source_coordinate_space": coordinate_space, + } + if coordinate_space != "physical_screen" or not hwnd: + meta["uia_coordinates"] = {"x": int(x), "y": int(y)} + return int(x), int(y), meta + try: + # UIAutomation providers do not necessarily use the same coordinate + # space as a DPI-aware WH_MOUSE_LL hook. In particular WeCom can + # expose a 1440x900 UIA root while the hook and DWM both report a + # 2880x1800 physical window. Comparing GetWindowRect with DWM alone + # therefore incorrectly produces a scale of 1. Prefer the actual + # UIA root rectangle as the destination coordinate space. + physical = wintypes.RECT() + hr = ctypes.windll.dwmapi.DwmGetWindowAttribute( + int(hwnd), 9, ctypes.byref(physical), ctypes.sizeof(physical) + ) + if hr == 0: + try: + import uiautomation as auto + + root = auto.ControlFromHandle(int(hwnd)) + rr = root.BoundingRectangle if root is not None else None + rw = int(rr.right - rr.left) if rr is not None else 0 + rh = int(rr.bottom - rr.top) if rr is not None else 0 + pw = int(physical.right - physical.left) + ph = int(physical.bottom - physical.top) + if min(rw, rh, pw, ph) > 0: + ux = round(int(rr.left) + (int(x) - physical.left) * rw / pw) + uy = round(int(rr.top) + (int(y) - physical.top) * rh / ph) + sx = pw / rw + sy = ph / rh + meta.update( + { + "uia_coordinates": {"x": ux, "y": uy}, + "dpi_scale": {"x": round(sx, 4), "y": round(sy, 4)}, + "coordinate_conversion": "dwm_physical_to_uia_root", + } + ) + return ux, uy, meta + except Exception as exc: + meta["uia_root_coordinate_error"] = f"{type(exc).__name__}: {exc}" + + logical = wintypes.RECT() + ctypes.windll.user32.GetWindowRect(int(hwnd), ctypes.byref(logical)) + # Compatibility fallback for providers whose root rectangle cannot + # be queried. DWMWA_EXTENDED_FRAME_BOUNDS is in physical pixels. + lw = int(logical.right - logical.left) + lh = int(logical.bottom - logical.top) + pw = int(physical.right - physical.left) + ph = int(physical.bottom - physical.top) + if hr == 0 and min(lw, lh, pw, ph) > 0: + sx = pw / lw + sy = ph / lh + ux = round(logical.left + (int(x) - physical.left) / sx) + uy = round(logical.top + (int(y) - physical.top) / sy) + meta.update( + { + "uia_coordinates": {"x": ux, "y": uy}, + "dpi_scale": {"x": round(sx, 4), "y": round(sy, 4)}, + "coordinate_conversion": "dwm_physical_to_getwindowrect_logical", + } + ) + return ux, uy, meta + except Exception as exc: + meta["coordinate_conversion_error"] = f"{type(exc).__name__}: {exc}" + meta["uia_coordinates"] = {"x": int(x), "y": int(y)} + meta["coordinate_conversion"] = "identity_fallback" + return int(x), int(y), meta + + @staticmethod + def _target_quality(target: dict[str, Any]) -> tuple[int, int, int, int, int]: + bounds = dict(target.get("bounds") or {}) + width = max(0, int(bounds.get("width") or 0)) + height = max(0, int(bounds.get("height") or 0)) + area = width * height + control_type = str(target.get("control_type") or "") + text = " ".join( + str(target.get(key) or "") + for key in ("name", "automation_id", "class_name", "control_type") + ).lower() + oversized = control_type == "WindowControl" or area > 1_000_000 + interactive = control_type in { + "ButtonControl", "EditControl", "TreeItemControl", "ListItemControl", + "MenuItemControl", "CheckBoxControl", "RadioButtonControl", + "ComboBoxControl", "TabItemControl", "HyperlinkControl", + } + semantic = any( + token in text + for token in ( + "search", "edit", "input", "message", "send", "contact", + "buddy", "result", "comment", "reply", + ) + ) + return ( + 0 if oversized else 1, + 1 if interactive else 0, + 1 if semantic else 0, + len(target.get("parent_chain") or []), + -area, + ) + + def _probe_mixed_uia_coordinates( + self, + hwnd: int, + raw_x: int, + raw_y: int, + *, + current_point: tuple[int, int], + current_target: dict[str, Any], + ) -> tuple[dict[str, Any], int, int, dict[str, Any]]: + """Resolve providers whose root and descendants use different DPI spaces. + + WeCom exposes a physical-size root but logical-size descendants. Only + probe alternative points when the direct hit is oversized and belongs + to a known mixed-space window, so standard desktop applications retain + their normal UIA hit-testing path. + """ + scope = dict(current_target.get("window_scope") or {}) + window_class = str(scope.get("class_name") or current_target.get("class_name") or "") + if ( + not self._is_root_or_oversized(current_target) + or window_class not in {"WeWorkWindow", "SearchResultWindow2"} + ): + return current_target, current_point[0], current_point[1], {} + + candidates: list[tuple[str, int, int, float]] = [ + ("direct", int(current_point[0]), int(current_point[1]), 1.0) + ] + try: + get_dpi = getattr(ctypes.windll.user32, "GetDpiForWindow", None) + dpi = int(get_dpi(int(hwnd)) or 96) if get_dpi else 96 + scale = max(1.0, float(dpi) / 96.0) + if scale > 1.05: + candidates.append( + ( + "window_dpi_absolute", + round(int(raw_x) / scale), + round(int(raw_y) / scale), + scale, + ) + ) + except Exception: + pass + + unique: list[tuple[str, int, int, float]] = [] + seen: set[tuple[int, int]] = set() + for item in candidates: + point = (item[1], item[2]) + if point not in seen: + seen.add(point) + unique.append(item) + + expected_pid = int(scope.get("pid") or 0) + expected_class = window_class + best_target = current_target + best_point = current_point + best_label = "direct" + best_scale = 1.0 + best_quality = self._target_quality(current_target) + attempts: list[dict[str, Any]] = [] + for label, px, py, scale in unique[1:]: + candidate, _err = self._element_from_point(px, py) + candidate_scope = dict((candidate or {}).get("window_scope") or {}) + same_scope = bool( + candidate + and (not expected_pid or int(candidate_scope.get("pid") or 0) == expected_pid) + and ( + not expected_class + or str(candidate_scope.get("class_name") or "") == expected_class + ) + ) + quality = self._target_quality(candidate or {}) if same_scope else (-1, 0, 0, 0, 0) + attempts.append( + { + "strategy": label, + "x": px, + "y": py, + "scale": round(scale, 4), + "target": str((candidate or {}).get("name") or ""), + "control_type": str((candidate or {}).get("control_type") or ""), + "same_scope": same_scope, + } + ) + if same_scope and quality > best_quality: + best_target = candidate + best_point = (px, py) + best_label = label + best_scale = scale + best_quality = quality + + return ( + best_target, + int(best_point[0]), + int(best_point[1]), + { + "uia_coordinate_candidates": attempts, + "coordinate_selection": best_label, + "mixed_coordinate_scale": round(best_scale, 4), + }, + ) + + def _on_key_down(self, packet: dict[str, Any]) -> None: + key = _SPECIAL_KEYS.get(int(packet.get("vk") or 0), "") + # Never call UIA here. Physical keys can arrive every few milliseconds; + # this consumer must remain faster than the hook producer even when the + # target accessibility provider is frozen for seconds. + target = self._cached_focus_snapshot() + if not self._is_text_entry(target): + fallback = dict(self._last_text_entry_target or {}) + if fallback and (not target or self._last_point_matches_target(fallback)): + target = fallback + if self._ime_composing: + # Enter/Escape inside an IME candidate/composition window confirms + # or cancels composition; it is not a business-level key action. + # Keep the transaction attached to the last real editor. + if target: + self._typing_active = True + self._last_key_at = time.monotonic() + self._emit_typing_activity(target, packet) + self._uia_request.set() + return + if not key: + self._typing_active = True + self._last_key_at = time.monotonic() + self._emit_typing_activity(target, packet) + self._uia_request.set() + return + if key in {"Backspace", "Delete"} and self._is_text_entry(target): + # Editing keys are part of the resulting value transaction. The + # next value poll records one complete fill instead of key spam. + self._typing_active = True + self._last_key_at = time.monotonic() + self._emit_typing_activity(target, packet) + self._uia_request.set() + return + # Enter/Tab/click are transaction boundaries: publish the latest + # sampled value before the boundary action, preserving causal order. + self._emit_cached_value_change(force=True) + self.emit( + RawUserEvent( + kind="key", + surface="windows", + key=key, + target=target, + meta={ + "input_backend": self.health().get("backend"), + "injected": bool(packet.get("injected")), + "source_time_ms": packet.get("source_time_ms"), + }, + ) + ) + self._uia_request.set() + + def _emit_typing_activity( + self, target: dict[str, Any], packet: dict[str, Any] + ) -> None: + """Journal one marker per text transaction without persisting key text. + + The marker lets diagnostics distinguish "no typing happened" from + "typing happened but the provider never exposed a value change". It + is intentionally not reduced into an Action. + """ + target = dict(target or self._last_text_entry_target or {}) + identity = "|".join( + str(target.get(key) or "") + for key in ("automation_id", "name", "control_type") + ) + scope = dict(target.get("window_scope") or {}) + identity = f"{scope.get('hwnd') or 0}|{identity}" + now = time.monotonic() + if ( + identity == self._last_typing_activity_identity + and now - self._last_typing_activity_at < 0.8 + ): + self._last_typing_activity_at = now + return + self._last_typing_activity_identity = identity + self._last_typing_activity_at = now + self.emit( + RawUserEvent( + kind="typing_activity", + surface="windows", + target=target, + meta={ + "input_backend": self.health().get("backend"), + "injected": bool(packet.get("injected")), + "source_time_ms": packet.get("source_time_ms"), + "content_recorded": False, + "unresolved_target": not bool(target), + "root_hwnd": int(self._last_root_hwnd or 0), + "last_uia_point": ( + {"x": self._last_uia_point[0], "y": self._last_uia_point[1]} + if self._last_uia_point else {} + ), + }, + ) + ) + + def _focused_control_snapshot(self) -> dict[str, Any]: + try: + import desktop_uia as uia + uia.ensure_com_sta() + import uiautomation as auto + ctrl = auto.GetFocusedControl() + info = self._control_snapshot(ctrl) if ctrl is not None else {} + self._ime_composing = self._is_transient_input_target(info) + unusable = self._is_root_or_oversized(info) or self._is_transient_input_target(info) + if unusable and self._last_uia_point: + try: + point_ctrl = auto.ControlFromPoint(*self._last_uia_point) + point_info = self._control_snapshot( + point_ctrl, x=self._last_uia_point[0], y=self._last_uia_point[1] + ) + if not self._is_root_or_oversized(point_info): + info = point_info + except Exception: + pass + if ( + (self._is_root_or_oversized(info) or self._is_transient_input_target(info)) + and self._last_raw_point + and self._last_uia_point + and self._last_root_hwnd + ): + try: + probed, _x, _y, _meta = self._probe_mixed_uia_coordinates( + self._last_root_hwnd, + self._last_raw_point[0], + self._last_raw_point[1], + current_point=self._last_uia_point, + current_target=info, + ) + if not self._is_root_or_oversized(probed) and not self._is_transient_input_target(probed): + info = probed + except Exception: + pass + if ( + (self._is_root_or_oversized(info) or self._is_transient_input_target(info)) + and self._last_text_entry_target + ): + refreshed = self._snapshot_live_text_control() + if refreshed: + info = refreshed + if ( + (self._is_root_or_oversized(info) or self._is_transient_input_target(info)) + and self._last_actionable_target + and self._last_point_matches_target(self._last_actionable_target) + ): + current_scope = dict(info.get("window_scope") or {}) + last_scope = dict(self._last_actionable_target.get("window_scope") or {}) + current_hwnd = int(current_scope.get("hwnd") or self._last_root_hwnd or 0) + last_hwnd = int(last_scope.get("hwnd") or 0) + if not current_hwnd or not last_hwnd or current_hwnd == last_hwnd: + info = dict(self._last_actionable_target) + if ( + (self._is_root_or_oversized(info) or self._is_transient_input_target(info)) + and self._last_uia_point + ): + scope = dict(info.get("window_scope") or {}) + class_name = str(scope.get("class_name") or "") + points = [self._last_uia_point] + if self._last_raw_point: + rx, ry = self._last_raw_point + points.extend( + [(rx, ry)] + + [(round(rx / scale), round(ry / scale)) for scale in (1.25, 1.5, 1.75, 2.0, 2.5, 3.0)] + ) + atlas_target = self._atlas_hit(class_name, points) + if atlas_target and self._is_text_entry(atlas_target): + atlas_target["window_scope"] = dict(scope) + self._bind_text_target(atlas_target) + info = self._snapshot_live_text_control() or atlas_target + if ( + (self._is_root_or_oversized(info) or self._is_transient_input_target(info)) + and self._last_text_entry_target + ): + current_scope = dict(info.get("window_scope") or {}) + editor_scope = dict(self._last_text_entry_target.get("window_scope") or {}) + current_pid = int(current_scope.get("pid") or 0) + editor_pid = int(editor_scope.get("pid") or 0) + # IME windows have a different HWND but normally share the + # application's PID. Never let the IME HWND replace the + # business editor just because the provider probe is cold. + if not current_pid or not editor_pid or current_pid == editor_pid: + refreshed = self._snapshot_live_text_control() + info = refreshed or dict(self._last_text_entry_target) + if info and not self._is_root_or_oversized(info) and not self._is_transient_input_target(info): + self._last_actionable_target = dict(info) + if self._is_text_entry(info): + self._bind_text_target(info, resolve_live=False) + return info + # A transient OS surface must never cross the recorder boundary as + # an application target. + return {} + except Exception as exc: + self._record_error(exc) + return {} + + def _on_mouse_up(self) -> None: + # Click already emitted on down (capture before UI disappears). + self._pending_target = {} + + def _poll_focus_value(self) -> None: + try: + info = self._focused_control_snapshot() + if not info: + return + # Skip if focused control belongs to Inspector + hwnd = int(info.get("window_scope", {}).get("hwnd") or 0) + try: + exclude = set(self.exclude_hwnds() or set()) + except Exception: + exclude = set() + if hwnd and hwnd in exclude: + return + fp = "|".join( + [ + str(info.get("automation_id") or ""), + str(info.get("name") or ""), + str(info.get("control_type") or ""), + ] + ) + val = str(info.get("value") or "") + if fp == self._last_focus_fp and val == self._last_focus_value: + return + if self._last_focus_fp and fp == self._last_focus_fp and val != self._last_focus_value: + self.emit( + RawUserEvent( + kind="value_change", + surface="windows", + text=val, + target=info, + meta={"input_type": "password" if "password" in (info.get("name") or "").lower() else ""}, + ) + ) + self._last_focus_fp = fp + self._last_focus_value = val + info["fingerprint"] = fp + except Exception: + return + + @staticmethod + def _is_text_entry(info: dict[str, Any]) -> bool: + text = " ".join( + str(info.get(key) or "") + for key in ("name", "automation_id", "class_name", "control_type") + ).lower() + return str(info.get("control_type") or "") == "EditControl" or any( + token in text + for token in ("edit", "input", "search", "compose", "message", "rich") + ) + + @staticmethod + def _is_transient_input_target(info: dict[str, Any]) -> bool: + text = " ".join( + str(value or "") + for value in ( + info.get("name"), + info.get("class_name"), + (info.get("window_scope") or {}).get("class_name"), + (info.get("window_scope") or {}).get("title"), + ) + ).lower() + return any( + token in text + for token in ( + "sopy_comp", "msctfime", "candidatewindow", "ime candidate", + "inputmethod", "input method", "compositionwindow", + "拼音", "候选词", "输入法", + ) + ) + + def _last_point_matches_target(self, target: dict[str, Any]) -> bool: + """Guard stale focus fallback with the last click's geometry.""" + bounds = dict(target.get("bounds") or {}) + try: + left = int(bounds.get("left")) + top = int(bounds.get("top")) + right = int(bounds.get("right")) + bottom = int(bounds.get("bottom")) + except (TypeError, ValueError): + return False + points: list[tuple[int, int]] = [] + if self._last_uia_point: + points.append(self._last_uia_point) + if self._last_raw_point: + rx, ry = self._last_raw_point + points.append((rx, ry)) + # Mixed-DPI providers commonly expose descendants at one of these + # Windows scale factors even when their root uses physical pixels. + for scale in (1.25, 1.5, 1.75, 2.0, 2.5, 3.0): + points.append((round(rx / scale), round(ry / scale))) + return any(left <= x <= right and top <= y <= bottom for x, y in points) + + def _seed_focus_baseline(self, info: dict[str, Any]) -> None: + """Remember the pre-input value at click time. + + Without this seed, fast typing can finish before the first 400 ms focus + poll and the completed text is mistaken for the initial baseline. + """ + if not info or not self._is_text_entry(info): + return + self._bind_text_target(info) + self._last_focus_fp = "|".join( + [ + str(info.get("automation_id") or ""), + str(info.get("name") or ""), + str(info.get("control_type") or ""), + ] + ) + self._last_focus_value = str(info.get("value") or "") + + def learn_elements( + self, + rows: list[dict[str, Any]], + *, + window: Optional[dict[str, Any]] = None, + ) -> None: + """Publish a bounded, value-free element atlas to the event-time path.""" + win = dict(window or {}) + class_name = str(win.get("class_name") or win.get("chosen_class") or "") + if not class_name: + return + cleaned: list[dict[str, Any]] = [] + for row in rows or []: + item = dict(row or {}) + bounds = dict(item.get("bounds") or item.get("rect") or {}) + if not bounds: + continue + item["bounds"] = bounds + item.pop("rect", None) + # Historical atlases are locators, not a store of user content. + item["value"] = "" + item["window_scope"] = { + "hwnd": int(win.get("hwnd") or 0), + "pid": int(win.get("pid") or 0), + "title": str(win.get("title") or ""), + "class_name": class_name, + } + cleaned.append(item) + if cleaned: + with self._atlas_lock: + self._element_atlas[class_name] = cleaned[-2500:] + + def _atlas_hit( + self, class_name: str, points: list[tuple[int, int]] + ) -> dict[str, Any]: + with self._atlas_lock: + rows = list(self._element_atlas.get(class_name) or []) + candidates: list[tuple[tuple[int, int, int, int], dict[str, Any]]] = [] + for row in rows: + bounds = dict(row.get("bounds") or {}) + try: + hit = any( + int(bounds.get("left")) <= x <= int(bounds.get("right")) + and int(bounds.get("top")) <= y <= int(bounds.get("bottom")) + for x, y in points + ) + except (TypeError, ValueError): + continue + if not hit or row.get("is_offscreen") is True: + continue + name = str(row.get("name") or row.get("automation_id") or "").lower() + control_type = str(row.get("control_type") or "") + width = max(0, int(bounds.get("width") or 0)) + height = max(0, int(bounds.get("height") or 0)) + area = width * height + if not area or width < 4 or height < 4: + continue + exact_editor = name in { + "searchedit", "send_rich_edit", "search_box", "compose_box" + } + editor = self._is_text_entry(row) and not name.endswith("_layout") + semantic = any( + token in name + for token in ("search_name", "contact_name", "send", "edit", "input") + ) + candidates.append( + ((1 if exact_editor else 0, 1 if editor else 0, 1 if semantic else 0, -area), row) + ) + return dict(max(candidates, default=((0, 0, 0, 0), {}), key=lambda x: x[0])[1]) + + def _bind_text_target( + self, info: dict[str, Any], *, resolve_live: bool = True + ) -> None: + if not info or not self._is_text_entry(info): + return + self._last_text_entry_target = dict(info) + if not resolve_live or not self._last_root_hwnd: + return + name = str(info.get("name") or "").strip() + if not name: + return + try: + import uiautomation as auto + + root = auto.ControlFromHandle(int(self._last_root_hwnd)) + candidate = root.Control(searchDepth=24, Name=name) + if candidate.Exists(0.35, 0.02): + self._live_text_control = candidate + except Exception: + self._live_text_control = None + + def _snapshot_live_text_control(self) -> dict[str, Any]: + if self._live_text_control is None: + return {} + try: + info = self._control_snapshot(self._live_text_control) + if info and self._is_text_entry(info): + return info + except Exception: + pass + self._live_text_control = None + return {} + + @staticmethod + def _is_root_or_oversized(info: dict[str, Any]) -> bool: + bounds = dict(info.get("bounds") or {}) + area = int(bounds.get("width") or 0) * int(bounds.get("height") or 0) + return str(info.get("control_type") or "") == "WindowControl" or area > 1_000_000 + + def _element_from_point(self, x: int, y: int) -> tuple[dict[str, Any], str]: + """Minimal directed read — NO ensure_ui_ready / full warm.""" + try: + import desktop_uia as uia + + uia.ensure_com_sta() + import uiautomation as auto + + ctrl = auto.ControlFromPoint(int(x), int(y)) + if ctrl is None: + return {}, "ControlFromPoint returned None" + return self._control_snapshot(ctrl, x=x, y=y), "" + except Exception as exc: + return {}, f"{type(exc).__name__}: {exc}" + + def _element_from_handle(self, hwnd: int) -> tuple[dict[str, Any], str]: + try: + import desktop_uia as uia + + uia.ensure_com_sta() + import uiautomation as auto + + ctrl = auto.ControlFromHandle(int(hwnd)) + if ctrl is None: + return {}, "ControlFromHandle returned None" + return self._control_snapshot(ctrl), "" + except Exception as exc: + return {}, f"{type(exc).__name__}: {exc}" + + @staticmethod + def _is_decorative_window_class(class_name: str) -> bool: + lowered = str(class_name or "").strip().lower() + return lowered in { + "shell_traywnd", "shell_secondarytraywnd", "progman", "workerw", + "ime", "msctfime ui", "tooltips_class32", "sopy_comp", + "perryshadowwnd", + } or any( + token in lowered + for token in ("shadowwnd", "dropshadow", "drop_shadow", "windowshadow") + ) + + @staticmethod + def _top_level_window_at_point( + x: int, + y: int, + *, + exclude_hwnds: set[int], + preferred_pid: int = 0, + ) -> int: + """Return the smallest visible top-level overlay containing a point. + + This is a generic Win32 fallback for applications whose accessibility + provider reports the owner window from ``WindowFromPoint`` while an + owned popup/list is visibly on top. + """ + if os.name != "nt": + return 0 + user32 = ctypes.windll.user32 + candidates: list[tuple[int, int]] = [] + @ctypes.WINFUNCTYPE(ctypes.c_bool, wintypes.HWND, wintypes.LPARAM) + def callback(hwnd, _lparam): + handle = int(hwnd or 0) + if not handle or handle in exclude_hwnds or not user32.IsWindowVisible(handle): + return True + pid_ref = ctypes.c_ulong() + user32.GetWindowThreadProcessId(handle, ctypes.byref(pid_ref)) + pid = int(pid_ref.value or 0) + if pid == os.getpid() or (preferred_pid and pid != preferred_pid): + return True + buf = ctypes.create_unicode_buffer(256) + user32.GetClassNameW(handle, buf, len(buf)) + class_name = buf.value + if WindowsRecorder._is_decorative_window_class(class_name): + return True + rect = wintypes.RECT() + if not user32.GetWindowRect(handle, ctypes.byref(rect)): + return True + if not (rect.left <= int(x) <= rect.right and rect.top <= int(y) <= rect.bottom): + return True + width = max(0, int(rect.right - rect.left)) + height = max(0, int(rect.bottom - rect.top)) + if width < 8 or height < 8: + return True + candidates.append((width * height, handle)) + return True + + try: + user32.EnumWindows(callback, 0) + except Exception: + return 0 + return min(candidates, default=(0, 0))[1] + + def _control_snapshot( + self, ctrl: Any, *, x: Optional[int] = None, y: Optional[int] = None + ) -> dict[str, Any]: + def safe(fn, default=""): + try: + return fn() + except Exception: + return default + + name = str(safe(lambda: ctrl.Name, "") or "") + value = "" + try: + value = str(ctrl.GetValuePattern().Value) + except Exception: + value = str(safe(lambda: getattr(ctrl, "Value", ""), "") or "") + if not value: + try: + value = str(ctrl.GetTextPattern().DocumentRange.GetText(-1) or "") + except Exception: + pass + if not value: + try: + value = str(ctrl.GetLegacyIAccessiblePattern().Value or "") + except Exception: + pass + aid = str(safe(lambda: ctrl.AutomationId, "") or "") + ctype = str(safe(lambda: ctrl.ControlTypeName, "") or "") + class_name = str(safe(lambda: ctrl.ClassName, "") or "") + framework = str(safe(lambda: ctrl.FrameworkId, "") or "") + enabled = safe(lambda: bool(ctrl.IsEnabled), None) + offscreen = safe(lambda: bool(ctrl.IsOffscreen), None) + rect: dict[str, int] = {} + try: + r = ctrl.BoundingRectangle + rect = { + "left": int(r.left), + "top": int(r.top), + "right": int(r.right), + "bottom": int(r.bottom), + "width": int(r.right - r.left), + "height": int(r.bottom - r.top), + } + except Exception: + rect = {} + + parent_chain: list[dict[str, str]] = [] + try: + p = ctrl.GetParentControl() + depth = 0 + while p is not None and depth < 8: + parent_chain.append( + { + "name": str(getattr(p, "Name", "") or ""), + "automation_id": str(getattr(p, "AutomationId", "") or ""), + "control_type": str(getattr(p, "ControlTypeName", "") or ""), + "class_name": str(getattr(p, "ClassName", "") or ""), + } + ) + p = p.GetParentControl() + depth += 1 + except Exception: + pass + + hwnd = 0 + pid = 0 + wtitle = "" + wclass = "" + try: + hwnd = int(ctrl.NativeWindowHandle or 0) + if not hwnd: + # walk up for hwnd + for item in parent_chain: + del item + p = ctrl + for _ in range(12): + h = int(getattr(p, "NativeWindowHandle", 0) or 0) + if h: + hwnd = h + break + p = p.GetParentControl() + if p is None: + break + if hwnd: + import desktop_uia as uia + + info = uia.window_info_from_hwnd(hwnd) if hasattr(uia, "window_info_from_hwnd") else None + if info is None: + # fallback via GetWindowText + import ctypes + + user32 = ctypes.windll.user32 + buf = ctypes.create_unicode_buffer(512) + user32.GetWindowTextW(hwnd, buf, 512) + wtitle = buf.value + user32.GetClassNameW(hwnd, buf, 512) + wclass = buf.value + pid_c = ctypes.c_ulong() + user32.GetWindowThreadProcessId(hwnd, ctypes.byref(pid_c)) + pid = int(pid_c.value) + else: + wtitle = info.title + wclass = info.class_name + pid = info.pid + hwnd = info.hwnd + except Exception: + pass + + snap = { + "name": name, + "value": value, + "automation_id": aid, + "control_type": ctype, + "class_name": class_name, + "framework_id": framework, + "is_enabled": enabled, + "is_offscreen": offscreen, + "bounds": rect, + "parent_chain": parent_chain, + "window_scope": { + "hwnd": hwnd, + "pid": pid, + "title": wtitle, + "class_name": wclass, + }, + } + if x is not None: + snap["capture_point"] = {"x": x, "y": y} + return snap + + @staticmethod + def _cursor_pos(user32) -> tuple[int, int]: + import ctypes + from ctypes import wintypes + + class POINT(ctypes.Structure): + _fields_ = [("x", wintypes.LONG), ("y", wintypes.LONG)] + + pt = POINT() + user32.GetCursorPos(ctypes.byref(pt)) + return int(pt.x), int(pt.y) + + @staticmethod + def _make_point(x: int, y: int): + import ctypes + from ctypes import wintypes + + class POINT(ctypes.Structure): + _fields_ = [("x", wintypes.LONG), ("y", wintypes.LONG)] + + return POINT(int(x), int(y)) diff --git a/tools/ui_inspector/redaction.py b/tools/ui_inspector/redaction.py new file mode 100644 index 0000000..7fe4f64 --- /dev/null +++ b/tools/ui_inspector/redaction.py @@ -0,0 +1,206 @@ +"""Sensitive-data redaction for Inspector artifacts.""" + +from __future__ import annotations + +import re +from typing import Any, Optional + +# Field / attribute name hints (case-insensitive). +_SENSITIVE_NAME_RE = re.compile( + r"(password|passwd|pwd|token|secret|cookie|authorization|auth|otp|" + r"captcha|verify.?code|verification.?code|sms.?code|pin)", + re.I, +) + +_PHONE_RE = re.compile(r"(? bool: + return bool(_SENSITIVE_NAME_RE.search(name or "")) + + +def looks_sensitive_value(value: str) -> bool: + s = value or "" + if not s: + return False + if _PHONE_RE.search(s) or _EMAIL_RE.search(s) or _TOKENISH_RE.fullmatch(s.strip()): + return True + if "password" in s.lower() or "secret" in s.lower(): + return True + return False + + +def sanitize_artifact(value: Any, *, sensitive_hint: bool = False) -> Any: + """Recursively sanitize event/evidence data before it is persisted. + + ``redact_input_payload`` protects the normalized Action input, but recorder + events also contain copies of the same value in target snapshots, ancestry + and raw_event. All persisted copies must pass through one boundary. + """ + if isinstance(value, dict): + input_type = str(value.get("input_type") or value.get("type") or "").lower() + field_name = str( + value.get("name") + or value.get("field") + or value.get("automation_id") + or "" + ) + local_sensitive = bool( + sensitive_hint + or input_type == "password" + or is_sensitive_field_name(field_name) + ) + out: dict[str, Any] = {} + for key, item in value.items(): + key_sensitive = local_sensitive or is_sensitive_field_name(str(key)) + if key in { + "name", "automation_id", "class_name", "control_type", + "framework_id", "path_hint", "logical_name", + } and isinstance(item, str) and not key_sensitive: + # Machine identifiers can be long token-like strings (for + # example conversation_list_container) without being secrets. + # Still remove explicit personal data embedded in a label. + out[key] = _EMAIL_RE.sub(REDACTED, _PHONE_RE.sub(REDACTED, item)) + continue + if key in {"value", "text", "input", "typed", "content", "plaintext"}: + if item in (None, ""): + out[key] = item + elif key_sensitive or looks_sensitive_value(str(item)): + out[key] = REDACTED + else: + out[key] = redact_text(str(item)) + else: + out[key] = sanitize_artifact(item, sensitive_hint=key_sensitive) + if local_sensitive: + out["sensitive"] = True + return out + if isinstance(value, list): + return [sanitize_artifact(item, sensitive_hint=sensitive_hint) for item in value] + if isinstance(value, tuple): + return [sanitize_artifact(item, sensitive_hint=sensitive_hint) for item in value] + if isinstance(value, str): + if sensitive_hint or looks_sensitive_value(value): + return REDACTED + return redact_text(value) + return value + + +def redact_text(value: str, *, force: bool = False) -> str: + if value is None: + return "" + s = str(value) + if force: + return REDACTED + s = _PHONE_RE.sub(REDACTED, s) + s = _EMAIL_RE.sub(REDACTED, s) + return s + + +def redact_input_payload( + payload: dict[str, Any], + *, + field_name: str = "", + input_type: str = "", +) -> tuple[dict[str, Any], bool]: + """ + Return (redacted_copy, was_sensitive). + Never leave raw password/token in action_flow / catalog. + """ + sensitive = False + out = dict(payload or {}) + itype = (input_type or out.get("input_type") or "").lower() + fname = field_name or str(out.get("name") or out.get("field") or "") + + if itype == "password" or is_sensitive_field_name(fname): + sensitive = True + for key in ("text", "value", "input", "typed", "content"): + if key in out and out[key] not in (None, ""): + out[key] = REDACTED + out["redacted"] = True + return out, True + + for key in ("text", "value", "input", "typed", "content"): + if key not in out: + continue + raw = out.get(key) + if raw is None: + continue + s = str(raw) + if looks_sensitive_value(s) or is_sensitive_field_name(str(out.get("name") or "")): + sensitive = True + force = ("@" in s) or bool(_PHONE_RE.search(s)) or bool(_TOKENISH_RE.search(s)) + out[key] = redact_text(s, force=force) + if _TOKENISH_RE.fullmatch(s.strip()) or looks_sensitive_value(s): + out[key] = REDACTED + if sensitive: + out["redacted"] = True + return out, sensitive + + +def redact_element_fields(el: dict[str, Any]) -> tuple[dict[str, Any], bool]: + out = dict(el or {}) + sensitive = False + name = str(out.get("name") or out.get("automation_id") or "") + role = str(out.get("control_type") or out.get("role") or "") + input_type = str(out.get("input_type") or out.get("type") or "") + if input_type == "password" or is_sensitive_field_name(name): + sensitive = True + if out.get("value"): + out["value"] = REDACTED + if out.get("text"): + out["text"] = REDACTED + out["sensitive"] = True + return out, True + for key in ("value", "text", "name"): + if key not in out or out[key] in (None, ""): + continue + s = str(out[key]) + # Stable element names/automation ids are locator structure. Long + # machine identifiers (OperationAndSendMessageRegion, generated DOM + # ids, etc.) must not be destroyed merely because they look token-like. + # Personal data embedded in a visible name is still scrubbed. + if key == "name": + cleaned = _EMAIL_RE.sub(REDACTED, _PHONE_RE.sub(REDACTED, s)) + out[key] = cleaned + sensitive = sensitive or cleaned != s + elif looks_sensitive_value(s): + sensitive = True + out[key] = REDACTED + if sensitive: + out["sensitive"] = True + # Edit controls with password-ish class + if "password" in role.lower() or "password" in str(out.get("class_name") or "").lower(): + sensitive = True + out["value"] = REDACTED + out["sensitive"] = True + return out, sensitive + + +def redact_locatable_rows(rows: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], bool]: + any_sens = False + out: list[dict[str, Any]] = [] + for r in rows or []: + cleaned, sens = redact_element_fields(r if isinstance(r, dict) else {}) + any_sens = any_sens or sens + out.append(cleaned) + return out, any_sens + + +def screenshot_sensitivity_meta( + *, + may_contain_sensitive: bool, + save_sensitive_screenshots: bool = False, +) -> dict[str, Any]: + return { + "may_contain_sensitive_data": bool(may_contain_sensitive), + "save_sensitive_screenshots": bool(save_sensitive_screenshots), + "note": ( + "截图暂时无法可靠脱敏；敏感页请关闭截图或人工审查。" + if may_contain_sensitive + else "" + ), + } diff --git a/tools/ui_inspector/requirements.txt b/tools/ui_inspector/requirements.txt index f2da8b2..2deb2b3 100644 --- a/tools/ui_inspector/requirements.txt +++ b/tools/ui_inspector/requirements.txt @@ -1,5 +1,4 @@ uiautomation>=2.0.18 -# 可选：导出 screen.png 需要 Pillow；没有也能 dump JSON -Pillow>=10.0.0 -# 企微自动暖树：与技能同款 ensure_ui_ready（请用宿主共享 venv 或本机安装） -# jiangchang-platform-kit>=1.2.5 +# 浏览器整页 dump（与技能共用宿主环境；本地开发可 editable 安装） +jiangchang-platform-kit>=1.2.11 +# playwright 由宿主 / kit 环境提供，勿在技能目录 playwright install diff --git a/tools/ui_inspector/skip_policy.py b/tools/ui_inspector/skip_policy.py new file mode 100644 index 0000000..9848d1a --- /dev/null +++ b/tools/ui_inspector/skip_policy.py @@ -0,0 +1,89 @@ +"""Decide whether a dump is blank/empty-shell and should not become a flow step. + +Only applied to browser surface. Does NOT skip same-URL scrolls +(content may change without URL change). +""" + +from __future__ import annotations + +from typing import Any, Optional + +import target + +# Locatable below this counts as "no useful candidates" +_MIN_LOCATABLE = 3 +# Tree/index tiny enough to treat as empty shell +_MAX_EMPTY_NODES = 2 + + +def _norm_url(url: str) -> str: + return (url or "").strip() + + +def is_blankish_url(url: str) -> bool: + u = _norm_url(url).lower() + if not u: + return True + if u == "about:blank" or u.startswith("about:blank?"): + return True + if u in {"about:srcdoc", "about:newtab"}: + return True + if u.startswith("chrome://newtab") or u.startswith("chrome://new-tab-page"): + return True + if u.startswith("edge://newtab") or u.startswith("edge://new-tab-page"): + return True + return False + + +def should_skip_empty_dump( + *, + surface: str, + url: str = "", + locatable_count: Optional[int] = None, + nodes_kept: Optional[int] = None, + sparse: bool = False, + locatable: Optional[list[dict[str, Any]]] = None, + stats: Optional[dict[str, Any]] = None, +) -> Optional[str]: + """Return skip reason code, or None if the dump should be kept. + + Safe rules (browser only): + - blank/newtab URL **and** almost no locatable → skip + - tiny tree + sparse/blankish + almost no locatable → skip + + A page with about:blank URL but rich locatable (nav race) is **kept**. + """ + if surface != target.SURFACE_BROWSER: + return None + + st = stats or {} + if locatable_count is None: + if locatable is not None: + locatable_count = len(locatable) + else: + locatable_count = int(st.get("locatable_count") or 0) + if nodes_kept is None: + nodes_kept = int(st.get("nodes_kept") or st.get("findall_count") or 0) + if not sparse: + sparse = bool(st.get("uia_sparse")) + + loc_n = int(locatable_count or 0) + kept = int(nodes_kept or 0) + blankish = is_blankish_url(url) + + if blankish and loc_n < _MIN_LOCATABLE: + return "blank_or_newtab_empty" + + if loc_n < _MIN_LOCATABLE and kept <= _MAX_EMPTY_NODES and (sparse or blankish): + return "empty_shell" + + return None + + +def skip_reason_text(reason: str, *, url: str = "") -> str: + u = _norm_url(url) or "(empty)" + if reason == "blank_or_newtab_empty": + return f"跳过空页：url={u!r} 且可定位候选过少（blank/新标签页）。" + if reason == "empty_shell": + return f"跳过空壳页：url={u!r} 节点过少且 sparse。" + return f"跳过本拍：{reason} url={u!r}" diff --git a/tools/ui_inspector/smoke_check.py b/tools/ui_inspector/smoke_check.py index f50ad12..134854f 100644 --- a/tools/ui_inspector/smoke_check.py +++ b/tools/ui_inspector/smoke_check.py @@ -1,4 +1,4 @@ -"""Smoke test for flow document API + step labeling + kit_wake helpers (v0.8.0).""" +"""Smoke test for flow document API + surface routing + kit helpers (v0.10).""" from __future__ import annotations @@ -11,7 +11,30 @@ if str(_HERE) not in sys.path: sys.path.insert(0, str(_HERE)) import artifacts as art +import desktop_uia as uia +import kit_browser import kit_wake +import skip_policy +import target +import user_prefs +from surfaces.registry import get_driver +from drivers.mobile import MobileDriver +from jiangchang_skill_core.rpa.page_dump import build_page_dump_result + + +def _assert_selection_api() -> None: + assert callable(uia.window_info_under_cursor) + assert callable(uia.find_wecom_search_result_window) + assert callable(uia.resolve_capture_target) + try: + _info, _cands, meta = uia.resolve_capture_target( + under_cursor=False, + prefer_wecom_search_popup=False, + ) + assert "strategy" in meta + except RuntimeError as exc: + assert "未找到可拾取" in str(exc) + print("OK selection api") def _assert_kit_wake_detect() -> None: @@ -23,6 +46,204 @@ def _assert_kit_wake_detect() -> None: print("OK kit_wake detect") +def _assert_surface_routing() -> None: + assert ( + target.resolve_surface(process_name="chrome.exe", class_name="Chrome_WidgetWin_1") + == target.SURFACE_BROWSER + ) + assert ( + target.resolve_surface(process_name="msedge.exe", class_name="Chrome_WidgetWin_1") + == target.SURFACE_BROWSER + ) + assert ( + target.resolve_surface(process_name="WXWork.exe", class_name="WeWorkWindow") + == target.SURFACE_WINDOWS + ) + assert get_driver(target.SURFACE_WINDOWS).driver_name == "uiautomation" + assert get_driver(target.SURFACE_BROWSER).driver_name == "playwright" + mobile = MobileDriver(surface=target.SURFACE_ANDROID) + try: + mobile.dump(info=None, selection_meta={}) + raise AssertionError("mobile dump should raise") + except RuntimeError as exc: + assert "SURFACE_UNSUPPORTED" in str(exc) + print("OK surface routing") + + +def _assert_user_prefs() -> None: + with tempfile.TemporaryDirectory(prefix="ui_prefs_") as tmp: + path = Path(tmp) / "user_prefs.json" + # monkeypatch prefs path + old = user_prefs._PREFS_PATH + try: + user_prefs._PREFS_PATH = path + user_prefs.save_prefs(mode="windows", start_url="https://example.com") + loaded = user_prefs.load_prefs() + assert loaded["mode"] == "windows" + assert loaded["start_url"] == "https://example.com" + user_prefs.save_prefs(mode="browser", start_url="") + loaded_empty = user_prefs.load_prefs() + assert loaded_empty["start_url"] == "" + user_prefs.save_prefs(mode="mobile", start_url="https://a.test") + loaded2 = user_prefs.load_prefs() + assert loaded2["mode"] == target.SURFACE_ANDROID + finally: + user_prefs._PREFS_PATH = old + print("OK user_prefs") + + +def _assert_skip_policy() -> None: + # blank + empty locatable → skip + assert ( + skip_policy.should_skip_empty_dump( + surface="browser", + url="about:blank", + locatable_count=0, + nodes_kept=1, + sparse=True, + ) + == "blank_or_newtab_empty" + ) + # blank URL but rich content (nav race like step05) → keep + assert ( + skip_policy.should_skip_empty_dump( + surface="browser", + url="about:blank", + locatable_count=74, + nodes_kept=97, + sparse=False, + ) + is None + ) + # real page → keep + assert ( + skip_policy.should_skip_empty_dump( + surface="browser", + url="https://www.douyin.com/jingxuan", + locatable_count=150, + nodes_kept=160, + sparse=False, + ) + is None + ) + # windows never skipped by this policy + assert ( + skip_policy.should_skip_empty_dump( + surface="windows", + url="", + locatable_count=0, + nodes_kept=1, + sparse=True, + ) + is None + ) + # same URL with content is NOT skipped (scroll-safe) + assert ( + skip_policy.should_skip_empty_dump( + surface="browser", + url="https://www.douyin.com/jingxuan", + locatable_count=20, + nodes_kept=30, + sparse=False, + ) + is None + ) + print("OK skip_policy") + + +def _assert_active_page_resolve() -> None: + """Unit-level: visible tab wins over sticky pages[0].""" + + class _FakePage: + def __init__(self, url: str, visible: bool, *, closed: bool = False) -> None: + self.url = url + self._visible = visible + self._closed = closed + + def is_closed(self) -> bool: + return self._closed + + def evaluate(self, _script: str) -> str: + return "visible" if self._visible else "hidden" + + def title(self) -> str: + return self.url + + class _FakeCtx: + def __init__(self, pages: list) -> None: + self.pages = pages + + search = _FakePage("https://www.douyin.com/search/foo", visible=False) + detail = _FakePage("https://www.douyin.com/video/123", visible=True) + ctx = _FakeCtx([search, detail]) + + page, meta = kit_browser.resolve_active_page(ctx, preferred=search) + assert page is detail, (page.url, meta) + assert meta["resolve"] == "visibility" + assert meta["page_count"] == 2 + assert meta["visible_count"] == 1 + + # No visible → preferred if still open + both_hidden = _FakeCtx( + [ + _FakePage("https://a.test/1", visible=False), + _FakePage("https://a.test/2", visible=False), + ] + ) + preferred = both_hidden.pages[0] + page2, meta2 = kit_browser.resolve_active_page(both_hidden, preferred=preferred) + assert page2 is preferred + assert meta2["resolve"] == "preferred_fallback" + + # Closed preferred → first open + closed_pref = _FakePage("https://gone", visible=False, closed=True) + open_only = _FakePage("https://alive", visible=False) + page3, meta3 = kit_browser.resolve_active_page( + _FakeCtx([closed_pref, open_only]), preferred=closed_pref + ) + assert page3 is open_only + assert meta3["resolve"] == "first_open" + print("OK active page resolve") + + +def _assert_page_dump_kit() -> None: + if not kit_browser.kit_page_dump_available(): + raise RuntimeError( + "jiangchang_skill_core.rpa.page_dump 不可用；" + "请 pip install -e jiangchang-platform-kit>=1.2.11" + ) + result = build_page_dump_result( + url="https://example.com", + title="Example", + a11y_snapshot={ + "role": "RootWebArea", + "name": "Example", + "children": [{"role": "button", "name": "OK", "children": []}], + }, + dom_nodes=[ + { + "tag": "button", + "role": "button", + "id": "", + "name_attr": "", + "testid": "", + "aria_label": "OK", + "placeholder": "", + "title": "", + "alt": "", + "href": "", + "type": "submit", + "text": "OK", + "disabled": False, + "rect": {}, + } + ], + ) + assert result["stats"]["locatable_count"] >= 1 + assert result["locatable"][0].get("primary_locator") + print("OK page_dump kit builder") + + def _assert_label() -> None: msg_page = [ {"control_type": "TreeItemControl", "name": "groupbuddyname", "value": "某群"}, @@ -83,7 +304,13 @@ def _assert_label() -> None: def main() -> int: + _assert_selection_api() _assert_kit_wake_detect() + _assert_surface_routing() + _assert_user_prefs() + _assert_skip_policy() + _assert_active_page_resolve() + _assert_page_dump_kit() _assert_label() with tempfile.TemporaryDirectory(prefix="ui_flow_smoke_") as tmp: root = Path(tmp) @@ -111,7 +338,15 @@ def main() -> int: ) assert idx == 1 assert "陈德炼" in name, name - art.write_json(step_dir / "meta.json", {"ok": True}) + art.write_json( + step_dir / "meta.json", + art.build_meta( + window={"title": "企业微信", "class_name": "WeWorkWindow", "pid": 1, "hwnd": 2}, + scene="demo", + surface="windows", + driver="uiautomation", + ), + ) art.register_step( flow_dir, flow, @@ -120,9 +355,25 @@ def main() -> int: window={"title": "企业微信", "class_name": "WeWorkWindow", "pid": 1, "hwnd": 2}, stats={"findall_count": 10, "locatable_count": 5, "nodes_kept": 3, "uia_sparse": False}, label=label, + surface="windows", + ) + # browser-shaped step registration + blow = art.load_flow(flow_dir) + art.register_step( + flow_dir, + blow, + step_index=2, + step_name="02_browser-demo", + window={"title": "精选", "class_name": "PlaywrightPage", "pid": 3, "hwnd": 4}, + stats={"findall_count": 20, "locatable_count": 12, "nodes_kept": 20, "uia_sparse": False}, + label="浏览器精选", + surface="browser", + url="https://www.douyin.com/jingxuan", ) flow = art.end_flow(flow_dir, art.load_flow(flow_dir)) assert flow["status"] == "ended" + summary = (flow_dir / "SUMMARY.md").read_text(encoding="utf-8") + assert "surface: `browser`" in summary or "surface: `windows`" in summary print(f"OK flow={flow_dir.name} step={name}") print("OK ui_inspector flow smoke passed") return 0 diff --git a/tools/ui_inspector/snapshot_service.py b/tools/ui_inspector/snapshot_service.py new file mode 100644 index 0000000..56348e4 --- /dev/null +++ b/tools/ui_inspector/snapshot_service.py @@ -0,0 +1,400 @@ +"""Snapshot service: wrap SurfaceDriver.dump into deduped UIState assets.""" + +from __future__ import annotations + +import time +from dataclasses import asdict, is_dataclass +from pathlib import Path +from typing import Any, Optional + +from models.observation import UIObservation +from models.serialize import to_dict, write_model_json +from models.state import UIState, compute_state_fingerprint +from redaction import ( + redact_locatable_rows, + sanitize_artifact, + screenshot_sensitivity_meta, +) + + +def _dump_to_parts(dump: Any) -> dict[str, Any]: + if dump is None: + return {} + if isinstance(dump, dict): + return dict(dump) + # DumpResult dataclass or duck-typed dump objects (tests / drivers). + if is_dataclass(dump) and not isinstance(dump, type) or hasattr(dump, "surface"): + return { + "surface": getattr(dump, "surface", "") or "", + "driver": getattr(dump, "driver", "") or "", + "window": dict(getattr(dump, "window", None) or {}), + "tree": getattr(dump, "tree", None) or {}, + "index": list(getattr(dump, "index", None) or []), + "locatable": list(getattr(dump, "locatable", None) or []), + "stats": dict(getattr(dump, "stats", None) or {}), + "selection": dict(getattr(dump, "selection", None) or {}), + "url": getattr(dump, "url", "") or "", + "page_title": getattr(dump, "page_title", "") or "", + "screenshot_bytes": getattr(dump, "screenshot_bytes", None), + "use_hwnd_screenshot": bool(getattr(dump, "use_hwnd_screenshot", False)), + "locatable_text": getattr(dump, "locatable_text", "") or "", + "tree_text": getattr(dump, "tree_text", "") or "", + } + return {} + + +class SnapshotService: + """ + Creates UIState records with fingerprint dedupe. + + Reuses SurfaceDriver.dump(); does not reimplement browser/desktop dump. + """ + + def __init__( + self, + flow_dir: Path, + *, + save_sensitive_screenshots: bool = False, + persist_evidence: bool = True, + ) -> None: + self.flow_dir = Path(flow_dir) + self.states_dir = self.flow_dir / "states" + self.states_dir.mkdir(parents=True, exist_ok=True) + self.observations_dir = self.flow_dir / "observations" + self.observations_dir.mkdir(parents=True, exist_ok=True) + self.save_sensitive_screenshots = save_sensitive_screenshots + self.persist_evidence = persist_evidence + self._by_fp: dict[str, UIState] = {} + self._order: list[str] = [] + self.latest: Optional[UIState] = None + self.latest_observation: Optional[UIObservation] = None + self._observation_order: list[str] = [] + + def load_existing(self) -> None: + """Index existing state JSON files (best-effort).""" + if not self.states_dir.exists(): + return + for path in sorted(self.states_dir.glob("*.json")): + try: + import json + + data = json.loads(path.read_text(encoding="utf-8")) + st = UIState( + id=data.get("id") or path.stem, + surface=data.get("surface") or "windows", + driver=data.get("driver") or "", + captured_at=data.get("captured_at") or "", + fingerprint=data.get("fingerprint") or "", + previous_state_id=data.get("previous_state_id"), + change_reason=data.get("change_reason") or "", + app=data.get("app") or {}, + window=data.get("window") or {}, + url=data.get("url") or "", + page_title=data.get("page_title") or "", + tree_path=data.get("tree_path") or "", + index_path=data.get("index_path") or "", + locatable_path=data.get("locatable_path") or "", + screenshot_path=data.get("screenshot_path") or "", + stats=data.get("stats") or {}, + selection=data.get("selection") or {}, + duplicate_of=data.get("duplicate_of"), + observation_count=int(data.get("observation_count") or 1), + may_contain_sensitive_data=bool( + data.get("may_contain_sensitive_data") + ), + ) + if st.fingerprint: + self._by_fp[st.fingerprint] = st + if st.id not in self._order: + self._order.append(st.id) + self.latest = st + except Exception: + continue + for path in sorted(self.observations_dir.glob("*/meta.json")): + try: + import json + + data = json.loads(path.read_text(encoding="utf-8")) + obs = UIObservation( + **{k: v for k, v in data.items() if k in UIObservation.__dataclass_fields__} + ) + self._observation_order.append(obs.id) + self.latest_observation = obs + except Exception: + continue + + def capture_from_dump( + self, + dump: Any, + *, + change_reason: str = "", + force_persist: bool = False, + previous_state_id: Optional[str] = None, + redact: bool = True, + ) -> UIState: + parts = _dump_to_parts(dump) + locatable = list(parts.get("locatable") or []) + sensitive = False + if redact: + locatable, sensitive = redact_locatable_rows(locatable) + + window = dict(parts.get("window") or {}) + url = parts.get("url") or "" + page_title = parts.get("page_title") or "" + surface = parts.get("surface") or "windows" + driver = parts.get("driver") or "" + fp = compute_state_fingerprint( + surface=surface, + driver=driver, + url=url, + page_title=page_title, + window=window, + locatable=locatable, + app={ + "title": window.get("title"), + "class_name": window.get("class_name"), + "process_name": window.get("process_name"), + }, + ) + + existing = self._by_fp.get(fp) + if existing is not None: + existing.bump_observation() + existing.duplicate_of = None + # Update meta path only — do not rewrite full tree evidence. + write_model_json(self.states_dir / f"{existing.id}.json", existing) + self.latest = existing + self._persist_observation( + existing, + parts, + locatable=locatable, + sensitive=sensitive, + change_reason=change_reason or "capture", + ) + return existing + + prev = previous_state_id + if prev is None and self.latest is not None: + prev = self.latest.id + + state = UIState( + surface=surface, + driver=driver, + fingerprint=fp, + previous_state_id=prev, + change_reason=change_reason or "capture", + app={ + "title": window.get("title"), + "class_name": window.get("class_name"), + "process_name": window.get("process_name"), + "pid": window.get("pid"), + "hwnd": window.get("hwnd"), + }, + window=window, + url=url, + page_title=page_title, + stats=dict(parts.get("stats") or {}), + selection=dict(parts.get("selection") or {}), + locatable=locatable, + may_contain_sensitive_data=sensitive, + ) + state.ensure_fingerprint() + + should_write_evidence = force_persist or bool(change_reason) or self.persist_evidence + if should_write_evidence: + self._persist_evidence(state, parts, sensitive=sensitive) + + write_model_json(self.states_dir / f"{state.id}.json", state) + self._by_fp[fp] = state + self._order.append(state.id) + self.latest = state + self._persist_observation( + state, + parts, + locatable=locatable, + sensitive=sensitive, + change_reason=change_reason or "capture", + ) + return state + + def _persist_observation( + self, + state: UIState, + parts: dict[str, Any], + *, + locatable: list[dict[str, Any]], + sensitive: bool, + change_reason: str, + ) -> UIObservation: + stats = dict(parts.get("stats") or {}) + selection = dict(parts.get("selection") or {}) + active_page = dict(stats.get("active_page") or {}) + browser_session = dict(stats.get("browser_session") or {}) + obs = UIObservation( + state_id=state.id, + previous_observation_id=( + self.latest_observation.id if self.latest_observation else None + ), + change_reason=change_reason, + surface=state.surface, + driver=state.driver, + page_id=str( + active_page.get("page_id") + or browser_session.get("page_id") + or selection.get("page_id") + or "" + ), + document_id=str(selection.get("document_id") or ""), + frame_id=str(selection.get("frame_id") or ""), + url=state.url, + page_title=state.page_title, + stats=stats, + selection=selection, + may_contain_sensitive_data=sensitive, + ) + evidence_dir = self.observations_dir / obs.id + evidence_dir.mkdir(parents=True, exist_ok=True) + import artifacts as art + + art.write_json( + evidence_dir / "tree.json", + sanitize_artifact(parts.get("tree") or {}), + ) + clean_index = sanitize_artifact(list(parts.get("index") or [])) + art.write_jsonl(evidence_dir / "index.jsonl", list(clean_index or [])) + art.write_json(evidence_dir / "locatable.json", list(locatable or [])) + obs.tree_path = f"observations/{obs.id}/tree.json" + obs.index_path = f"observations/{obs.id}/index.jsonl" + obs.locatable_path = f"observations/{obs.id}/locatable.json" + + sensitivity_meta = screenshot_sensitivity_meta( + may_contain_sensitive=sensitive, + save_sensitive_screenshots=self.save_sensitive_screenshots, + ) + art.write_json(evidence_dir / "sensitivity.json", sensitivity_meta) + if (not sensitive) or self.save_sensitive_screenshots: + png = parts.get("screenshot_bytes") + if png: + try: + (evidence_dir / "screen.png").write_bytes(png) + obs.screenshot_path = f"observations/{obs.id}/screen.png" + except Exception: + pass + elif parts.get("use_hwnd_screenshot"): + hwnd = int((parts.get("window") or {}).get("hwnd") or 0) + if hwnd and art.capture_window_png(hwnd, evidence_dir / "screen.png"): + obs.screenshot_path = f"observations/{obs.id}/screen.png" + + write_model_json(evidence_dir / "meta.json", obs) + self.latest_observation = obs + self._observation_order.append(obs.id) + return obs + + def _persist_evidence( + self, + state: UIState, + parts: dict[str, Any], + *, + sensitive: bool, + ) -> None: + evidence_dir = self.states_dir / state.id + evidence_dir.mkdir(parents=True, exist_ok=True) + import artifacts as art + + art.write_json( + evidence_dir / "tree.json", + sanitize_artifact(parts.get("tree") or {}), + ) + clean_index = sanitize_artifact(list(parts.get("index") or [])) + art.write_jsonl(evidence_dir / "index.jsonl", list(clean_index or [])) + art.write_json(evidence_dir / "locatable.json", list(state.locatable or [])) + state.tree_path = f"states/{state.id}/tree.json" + state.index_path = f"states/{state.id}/index.jsonl" + state.locatable_path = f"states/{state.id}/locatable.json" + + sens_meta = screenshot_sensitivity_meta( + may_contain_sensitive=sensitive, + save_sensitive_screenshots=self.save_sensitive_screenshots, + ) + art.write_json(evidence_dir / "sensitivity.json", sens_meta) + + save_png = (not sensitive) or self.save_sensitive_screenshots + if save_png: + png = parts.get("screenshot_bytes") + if png: + try: + (evidence_dir / "screen.png").write_bytes(png) + state.screenshot_path = f"states/{state.id}/screen.png" + except Exception: + pass + elif parts.get("use_hwnd_screenshot"): + hwnd = int((parts.get("window") or {}).get("hwnd") or 0) + if hwnd: + ok = art.capture_window_png(hwnd, evidence_dir / "screen.png") + if ok: + state.screenshot_path = f"states/{state.id}/screen.png" + elif sensitive: + state.may_contain_sensitive_data = True + + def wait_until_stable( + self, + dump_fn, + *, + timeout_sec: float = 3.0, + poll_sec: float = 0.25, + change_reason: str = "after_action", + ) -> UIState: + """ + Poll dump_fn until two consecutive fingerprints match or timeout. + dump_fn() -> DumpResult / dump-like object + """ + deadline = time.monotonic() + max(0.2, float(timeout_sec)) + last_fp: Optional[str] = None + last_dump = None + while True: + last_dump = dump_fn() + parts = _dump_to_parts(last_dump) + locatable, _ = redact_locatable_rows(list(parts.get("locatable") or [])) + fp = compute_state_fingerprint( + surface=parts.get("surface") or "windows", + driver=parts.get("driver") or "", + url=parts.get("url") or "", + page_title=parts.get("page_title") or "", + window=parts.get("window") or {}, + locatable=locatable, + ) + if last_fp is not None and fp == last_fp: + break + last_fp = fp + if time.monotonic() >= deadline: + break + time.sleep(max(0.05, float(poll_sec))) + return self.capture_from_dump( + last_dump, + change_reason=change_reason, + force_persist=True, + ) + + def state_ids(self) -> list[str]: + return list(self._order) + + def observation_ids(self) -> list[str]: + return list(self._observation_order) + + def get(self, state_id: str) -> Optional[UIState]: + for st in self._by_fp.values(): + if st.id == state_id: + return st + path = self.states_dir / f"{state_id}.json" + if path.exists(): + import json + from models.state import UIState as S + + data = json.loads(path.read_text(encoding="utf-8")) + return S(**{k: v for k, v in data.items() if k in S.__dataclass_fields__}) + return None + + def to_summary_dict(self, state: UIState) -> dict[str, Any]: + d = to_dict(state) + d["locatable"] = list(state.locatable or []) + return d diff --git a/tools/ui_inspector/state_diff.py b/tools/ui_inspector/state_diff.py new file mode 100644 index 0000000..fa2627a --- /dev/null +++ b/tools/ui_inspector/state_diff.py @@ -0,0 +1,275 @@ +"""State Diff + proposed postconditions.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Optional + + +@dataclass +class StateChange: + kind: str + detail: dict[str, Any] = field(default_factory=dict) + confidence: float = 0.7 + + +@dataclass +class ProposedPostcondition: + description: str + kind: str + status: str = "proposed" # proposed | user_confirmed | runtime_verified + detail: dict[str, Any] = field(default_factory=dict) + + +def _row_key(r: dict[str, Any]) -> str: + return "|".join( + [ + str(r.get("control_type") or r.get("role") or ""), + str(r.get("automation_id") or ""), + str(r.get("name") or ""), + str(r.get("class_name") or ""), + ] + ) + + +def _row_value(r: dict[str, Any]) -> str: + return str(r.get("value") or r.get("text") or "") + + +def _index_rows(rows: Optional[list[dict[str, Any]]]) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + for r in rows or []: + if not isinstance(r, dict): + continue + k = _row_key(r) + if k.strip("|"): + out[k] = r + return out + + +def diff_states( + before: Optional[dict[str, Any]], + after: Optional[dict[str, Any]], +) -> list[StateChange]: + """ + Diff two state-like dicts (UIState.to_dict or lightweight summaries). + + Expected keys: url, page_title, window, locatable, stats, selection + """ + b = before or {} + a = after or {} + changes: list[StateChange] = [] + + b_url = str(b.get("url") or "") + a_url = str(a.get("url") or "") + if b_url != a_url: + changes.append( + StateChange( + kind="url_changed", + detail={"from": b_url, "to": a_url}, + ) + ) + + b_title = str(b.get("page_title") or (b.get("window") or {}).get("title") or "") + a_title = str(a.get("page_title") or (a.get("window") or {}).get("title") or "") + if b_title != a_title: + changes.append( + StateChange( + kind="title_changed", + detail={"from": b_title, "to": a_title}, + ) + ) + + b_cls = str((b.get("window") or {}).get("class_name") or "") + a_cls = str((a.get("window") or {}).get("class_name") or "") + b_hwnd = (b.get("window") or {}).get("hwnd") + a_hwnd = (a.get("window") or {}).get("hwnd") + if b_cls and a_cls and b_cls != a_cls: + changes.append( + StateChange( + kind="window_changed", + detail={"from_class": b_cls, "to_class": a_cls}, + ) + ) + if b_hwnd and not a_hwnd: + changes.append(StateChange(kind="window_disappeared", detail={"class_name": b_cls})) + if a_hwnd and not b_hwnd: + changes.append(StateChange(kind="window_appeared", detail={"class_name": a_cls})) + + # Popup heuristics by class name + for label, cls in (("before", b_cls), ("after", a_cls)): + del label + if "SearchResult" in a_cls and "SearchResult" not in b_cls: + changes.append( + StateChange(kind="popup_appeared", detail={"class_name": a_cls}) + ) + if "SearchResult" in b_cls and "SearchResult" not in a_cls: + changes.append( + StateChange(kind="popup_disappeared", detail={"class_name": b_cls}) + ) + + b_rows = _index_rows(b.get("locatable") or []) + a_rows = _index_rows(a.get("locatable") or []) + b_keys = set(b_rows) + a_keys = set(a_rows) + + for k in sorted(a_keys - b_keys)[:40]: + r = a_rows[k] + changes.append( + StateChange( + kind="element_appeared", + detail={ + "key": k, + "name": r.get("name"), + "value": _row_value(r), + "control_type": r.get("control_type") or r.get("role"), + "automation_id": r.get("automation_id"), + }, + ) + ) + for k in sorted(b_keys - a_keys)[:40]: + r = b_rows[k] + changes.append( + StateChange( + kind="element_disappeared", + detail={ + "key": k, + "name": r.get("name"), + "value": _row_value(r), + "control_type": r.get("control_type") or r.get("role"), + "automation_id": r.get("automation_id"), + }, + ) + ) + for k in sorted(b_keys & a_keys)[:80]: + bv = _row_value(b_rows[k]) + av = _row_value(a_rows[k]) + if bv != av: + changes.append( + StateChange( + kind="element_value_changed", + detail={ + "key": k, + "name": a_rows[k].get("name"), + "from": bv, + "to": av, + "automation_id": a_rows[k].get("automation_id"), + }, + ) + ) + + b_focus = (b.get("selection") or {}).get("focused") or (b.get("selection") or {}).get( + "focus" + ) + a_focus = (a.get("selection") or {}).get("focused") or (a.get("selection") or {}).get( + "focus" + ) + if b_focus != a_focus and (b_focus or a_focus): + changes.append( + StateChange( + kind="focus_changed", + detail={"from": b_focus, "to": a_focus}, + ) + ) + + b_loc = int((b.get("stats") or {}).get("locatable_count") or len(b.get("locatable") or [])) + a_loc = int((a.get("stats") or {}).get("locatable_count") or len(a.get("locatable") or [])) + if b_loc != a_loc: + changes.append( + StateChange( + kind="locatable_count_changed", + detail={"from": b_loc, "to": a_loc}, + ) + ) + + return changes + + +def propose_postconditions(changes: list[StateChange]) -> list[ProposedPostcondition]: + """Generate suggested postconditions (always status=proposed).""" + props: list[ProposedPostcondition] = [] + for ch in changes: + kind = ch.kind + d = ch.detail or {} + if kind == "url_changed": + props.append( + ProposedPostcondition( + description=f"URL became {d.get('to')!r}", + kind="url_equals", + detail=d, + ) + ) + elif kind == "popup_disappeared": + props.append( + ProposedPostcondition( + description=f"Window/popup {d.get('class_name')!r} disappeared", + kind="window_absent", + detail=d, + ) + ) + elif kind == "popup_appeared": + props.append( + ProposedPostcondition( + description=f"Window/popup {d.get('class_name')!r} appeared", + kind="window_present", + detail=d, + ) + ) + elif kind == "title_changed": + props.append( + ProposedPostcondition( + description=f"Title became {d.get('to')!r}", + kind="title_equals", + detail=d, + ) + ) + elif kind == "element_appeared": + name = d.get("name") or d.get("automation_id") or d.get("value") + props.append( + ProposedPostcondition( + description=f"Element {name!r} appeared", + kind="element_present", + detail=d, + ) + ) + elif kind == "element_disappeared": + name = d.get("name") or d.get("automation_id") or d.get("value") + props.append( + ProposedPostcondition( + description=f"Element {name!r} disappeared", + kind="element_absent", + detail=d, + ) + ) + elif kind == "element_value_changed": + props.append( + ProposedPostcondition( + description=( + f"Element {d.get('name') or d.get('automation_id')!r} " + f"value became {d.get('to')!r}" + ), + kind="element_value_equals", + detail=d, + ) + ) + return props + + +def changes_to_dicts(changes: list[StateChange]) -> list[dict[str, Any]]: + return [ + {"kind": c.kind, "detail": c.detail, "confidence": c.confidence} for c in changes + ] + + +def postconditions_to_dicts( + props: list[ProposedPostcondition], +) -> list[dict[str, Any]]: + return [ + { + "description": p.description, + "kind": p.kind, + "status": p.status, + "detail": p.detail, + } + for p in props + ] diff --git a/tools/ui_inspector/storage/__init__.py b/tools/ui_inspector/storage/__init__.py new file mode 100644 index 0000000..ed1d34f --- /dev/null +++ b/tools/ui_inspector/storage/__init__.py @@ -0,0 +1,6 @@ +"""Capture persistence API.""" + +from core.pipeline.event_journal import EventJournal + +__all__ = ["EventJournal"] + diff --git a/tools/ui_inspector/storage/capture_reader.py b/tools/ui_inspector/storage/capture_reader.py new file mode 100644 index 0000000..14e4bc8 --- /dev/null +++ b/tools/ui_inspector/storage/capture_reader.py @@ -0,0 +1,20 @@ +"""Stable capture reader used by diagnostics and downstream AI tooling.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +def read_json(path: Path) -> dict[str, Any]: + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def read_action_flow(flow_dir: Path) -> dict[str, Any]: + return read_json(Path(flow_dir) / "action_flow.json") + + +def read_element_catalog(flow_dir: Path) -> dict[str, Any]: + return read_json(Path(flow_dir) / "element_catalog.json") + diff --git a/tools/ui_inspector/storage/capture_writer.py b/tools/ui_inspector/storage/capture_writer.py new file mode 100644 index 0000000..a7715c6 --- /dev/null +++ b/tools/ui_inspector/storage/capture_writer.py @@ -0,0 +1,6 @@ +"""Compatibility export for the current atomic artifact writer.""" + +from artifacts import create_flow, load_flow, save_flow, write_json + +__all__ = ["create_flow", "load_flow", "save_flow", "write_json"] + diff --git a/tools/ui_inspector/surfaces/__init__.py b/tools/ui_inspector/surfaces/__init__.py new file mode 100644 index 0000000..cd3ed55 --- /dev/null +++ b/tools/ui_inspector/surfaces/__init__.py @@ -0,0 +1,2 @@ +"""Automation surface implementations.""" + diff --git a/tools/ui_inspector/surfaces/base.py b/tools/ui_inspector/surfaces/base.py new file mode 100644 index 0000000..788a49f --- /dev/null +++ b/tools/ui_inspector/surfaces/base.py @@ -0,0 +1,19 @@ +"""Surface contracts shared by browser, Windows, and future mobile drivers.""" + +from __future__ import annotations + +from typing import Any, Protocol + + +class SurfaceRecorder(Protocol): + status: str + def start(self) -> None: ... + def stop(self) -> None: ... + def pause(self) -> None: ... + def resume(self) -> None: ... + + +class SurfaceDumpDriver(Protocol): + surface: str + def dump(self, **kwargs: Any) -> Any: ... + diff --git a/tools/ui_inspector/surfaces/browser/__init__.py b/tools/ui_inspector/surfaces/browser/__init__.py new file mode 100644 index 0000000..b9e1ade --- /dev/null +++ b/tools/ui_inspector/surfaces/browser/__init__.py @@ -0,0 +1,6 @@ +"""Browser surface public API.""" + +from .recorder import BrowserRecorder +from .dump_driver import BrowserPlaywrightDriver + +__all__ = ["BrowserRecorder", "BrowserPlaywrightDriver"] diff --git a/tools/ui_inspector/surfaces/browser/dump_driver.py b/tools/ui_inspector/surfaces/browser/dump_driver.py new file mode 100644 index 0000000..736038d --- /dev/null +++ b/tools/ui_inspector/surfaces/browser/dump_driver.py @@ -0,0 +1,6 @@ +"""Browser full-dump driver export.""" + +from drivers.browser_pw import BrowserPlaywrightDriver + +__all__ = ["BrowserPlaywrightDriver"] + diff --git a/tools/ui_inspector/surfaces/browser/locator_compiler.py b/tools/ui_inspector/surfaces/browser/locator_compiler.py new file mode 100644 index 0000000..e3b2d79 --- /dev/null +++ b/tools/ui_inspector/surfaces/browser/locator_compiler.py @@ -0,0 +1,6 @@ +"""Browser locator compiler export.""" + +from locators.browser import compile_browser_locators + +__all__ = ["compile_browser_locators"] + diff --git a/tools/ui_inspector/surfaces/browser/locator_validator.py b/tools/ui_inspector/surfaces/browser/locator_validator.py new file mode 100644 index 0000000..a53395f --- /dev/null +++ b/tools/ui_inspector/surfaces/browser/locator_validator.py @@ -0,0 +1,6 @@ +"""Browser live locator validator export.""" + +from kit_browser import validate_element_on_page + +__all__ = ["validate_element_on_page"] + diff --git a/tools/ui_inspector/surfaces/browser/page_registry.py b/tools/ui_inspector/surfaces/browser/page_registry.py new file mode 100644 index 0000000..1a22a70 --- /dev/null +++ b/tools/ui_inspector/surfaces/browser/page_registry.py @@ -0,0 +1,6 @@ +"""Stable browser page/frame identity facade.""" + +from kit_browser import page_id_for + +__all__ = ["page_id_for"] + diff --git a/tools/ui_inspector/surfaces/browser/recorder.py b/tools/ui_inspector/surfaces/browser/recorder.py new file mode 100644 index 0000000..266a3ce --- /dev/null +++ b/tools/ui_inspector/surfaces/browser/recorder.py @@ -0,0 +1,6 @@ +"""Browser recorder compatibility export during staged migration.""" + +from recorder.browser import BrowserRecorder + +__all__ = ["BrowserRecorder"] + diff --git a/tools/ui_inspector/surfaces/browser/runtime.py b/tools/ui_inspector/surfaces/browser/runtime.py new file mode 100644 index 0000000..2983be9 --- /dev/null +++ b/tools/ui_inspector/surfaces/browser/runtime.py @@ -0,0 +1,23 @@ +"""Inspector-owned Playwright runtime facade.""" + +from kit_browser import ( + call_on_each_open_page, + call_on_page, + clear_event_bridge, + dump_current_page, + ensure_session, + install_event_bridge, + page_id_for, + screenshot_png, + session_info, + shutdown, + validate_element_on_page, +) + +__all__ = [ + "call_on_each_open_page", "call_on_page", "clear_event_bridge", + "dump_current_page", "ensure_session", "install_event_bridge", + "page_id_for", "screenshot_png", "session_info", "shutdown", + "validate_element_on_page", +] + diff --git a/tools/ui_inspector/surfaces/mobile/__init__.py b/tools/ui_inspector/surfaces/mobile/__init__.py new file mode 100644 index 0000000..170c18e --- /dev/null +++ b/tools/ui_inspector/surfaces/mobile/__init__.py @@ -0,0 +1,6 @@ +"""Future mobile surface public API.""" + +from drivers.mobile import MobileDriver + +__all__ = ["MobileDriver"] + diff --git a/tools/ui_inspector/surfaces/registry.py b/tools/ui_inspector/surfaces/registry.py new file mode 100644 index 0000000..c05e1a4 --- /dev/null +++ b/tools/ui_inspector/surfaces/registry.py @@ -0,0 +1,22 @@ +"""Resolve a dump driver through the canonical Surface namespace.""" + +from __future__ import annotations + +import target +from .browser.dump_driver import BrowserPlaywrightDriver +from .mobile import MobileDriver +from .windows.dump_driver import WindowsUiaDriver + + +def get_driver(surface: str): + if surface == target.SURFACE_BROWSER: + return BrowserPlaywrightDriver() + if surface == target.SURFACE_WINDOWS: + return WindowsUiaDriver() + if surface in {target.SURFACE_ANDROID, target.SURFACE_IOS}: + return MobileDriver(surface=surface) + raise RuntimeError(f"ERROR:SURFACE_UNSUPPORTED unknown surface={surface!r}") + + +__all__ = ["get_driver"] + diff --git a/tools/ui_inspector/surfaces/windows/__init__.py b/tools/ui_inspector/surfaces/windows/__init__.py new file mode 100644 index 0000000..dafcdd2 --- /dev/null +++ b/tools/ui_inspector/surfaces/windows/__init__.py @@ -0,0 +1,6 @@ +"""Windows surface public API.""" + +from .recorder import WindowsRecorder +from .dump_driver import WindowsUiaDriver + +__all__ = ["WindowsRecorder", "WindowsUiaDriver"] diff --git a/tools/ui_inspector/surfaces/windows/dump_driver.py b/tools/ui_inspector/surfaces/windows/dump_driver.py new file mode 100644 index 0000000..6284039 --- /dev/null +++ b/tools/ui_inspector/surfaces/windows/dump_driver.py @@ -0,0 +1,4 @@ +from drivers.windows_uia import WindowsUiaDriver + +__all__ = ["WindowsUiaDriver"] + diff --git a/tools/ui_inspector/surfaces/windows/locator_compiler.py b/tools/ui_inspector/surfaces/windows/locator_compiler.py new file mode 100644 index 0000000..4c99636 --- /dev/null +++ b/tools/ui_inspector/surfaces/windows/locator_compiler.py @@ -0,0 +1,4 @@ +from locators.windows import compile_windows_locators + +__all__ = ["compile_windows_locators"] + diff --git a/tools/ui_inspector/surfaces/windows/recorder.py b/tools/ui_inspector/surfaces/windows/recorder.py new file mode 100644 index 0000000..007b4cf --- /dev/null +++ b/tools/ui_inspector/surfaces/windows/recorder.py @@ -0,0 +1,4 @@ +from recorder.windows import WindowsRecorder + +__all__ = ["WindowsRecorder"] + diff --git a/tools/ui_inspector/target.py b/tools/ui_inspector/target.py new file mode 100644 index 0000000..9eadda2 --- /dev/null +++ b/tools/ui_inspector/target.py @@ -0,0 +1,82 @@ +"""Resolve capture surface from a Win32 window (windows | browser | mobile stub).""" + +from __future__ import annotations + +from typing import Any, Optional + +# Logical surfaces (align with RPA_ELEMENT_LOCATING.md) +SURFACE_WINDOWS = "windows" +SURFACE_BROWSER = "browser" +SURFACE_ANDROID = "android" +SURFACE_IOS = "ios" + +_BROWSER_PROCESS_NAMES = { + "chrome.exe", + "msedge.exe", + "chromium.exe", + "brave.exe", + "opera.exe", + "vivaldi.exe", +} + +_BROWSER_CLASS_HINTS = { + "Chrome_WidgetWin_1", + "Chrome_WidgetWin_0", + "Chrome_WidgetWin_2", +} + + +def normalize_process_name(name: str) -> str: + return (name or "").strip().lower() + + +def is_browser_window( + *, + process_name: str = "", + class_name: str = "", + title: str = "", +) -> bool: + proc = normalize_process_name(process_name) + if proc in _BROWSER_PROCESS_NAMES: + return True + cls = (class_name or "").strip() + if cls in _BROWSER_CLASS_HINTS: + return True + # Edge/Chrome sometimes expose empty process briefly; title heuristics last + t = title or "" + if " - Google Chrome" in t or " - Microsoft Edge" in t or t.endswith(" - Edge"): + return True + if t.endswith(" - Chrome"): + return True + return False + + +def resolve_surface( + *, + process_name: str = "", + class_name: str = "", + title: str = "", +) -> str: + """Return surface id for this top-level window.""" + if is_browser_window( + process_name=process_name, class_name=class_name, title=title + ): + return SURFACE_BROWSER + return SURFACE_WINDOWS + + +def surface_from_window_info(info: Any) -> str: + return resolve_surface( + process_name=getattr(info, "process_name", "") or "", + class_name=getattr(info, "class_name", "") or "", + title=getattr(info, "title", "") or "", + ) + + +def surface_label(surface: str) -> str: + return { + SURFACE_WINDOWS: "桌面 UIA", + SURFACE_BROWSER: "浏览器 Playwright", + SURFACE_ANDROID: "Android（未实现）", + SURFACE_IOS: "iOS（未实现）", + }.get(surface, surface) diff --git a/tools/ui_inspector/tests/__init__.py b/tools/ui_inspector/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/ui_inspector/tests/contract/__init__.py b/tools/ui_inspector/tests/contract/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tools/ui_inspector/tests/contract/__init__.py @@ -0,0 +1 @@ + diff --git a/tools/ui_inspector/tests/contract/test_capture_contract.py b/tools/ui_inspector/tests/contract/test_capture_contract.py new file mode 100644 index 0000000..8551b7c --- /dev/null +++ b/tools/ui_inspector/tests/contract/test_capture_contract.py @@ -0,0 +1,207 @@ +from pathlib import Path + +from flow_session import FlowSession +from recorder.base import RawUserEvent +from diagnostics.capture_validator import validate_capture +from diagnostics.replay_preflight import build_replay_preflight + + +def test_session_always_creates_raw_event_journal(tmp_path: Path) -> None: + session = FlowSession(tmp_path, flow_name="contract", surface="browser", dump_fn=None) + session.manager.start() + session.manager.status = "recording" + event = RawUserEvent(kind="click", surface="browser", target={"role": "button", "name": "Go"}) + session.journal.append(event, channel="semantic") + assert (tmp_path / "journal" / "raw_events.jsonl").exists() + + +def test_validator_flags_oversized_click_target(tmp_path: Path) -> None: + import json + actions = tmp_path / "actions" + actions.mkdir() + (tmp_path / "elements").mkdir() + (tmp_path / "observations").mkdir() + (tmp_path / "journal").mkdir() + (tmp_path / "journal" / "raw_events.jsonl").write_text("{}\n", encoding="utf-8") + (actions / "a.json").write_text( + json.dumps({ + "id": "a", "sequence": 1, "action_type": "click", + "target_snapshot": {"rect": {"width": 1200, "height": 5000}}, + "evidence": {"eventual_observation_id": "obs"}, + }), + encoding="utf-8", + ) + result = validate_capture(tmp_path) + assert any(issue["code"] == "CLICK_TARGET_OVERSIZED" for issue in result["issues"]) + + +def test_validator_flags_missing_desktop_input_and_generic_targets(tmp_path: Path) -> None: + import json + + (tmp_path / "actions").mkdir() + (tmp_path / "elements").mkdir() + (tmp_path / "observations").mkdir() + (tmp_path / "journal").mkdir() + journal = { + "event": { + "kind": "typing_activity", "surface": "windows", + "target": {"name": "searchedit", "control_type": "TreeControl"}, + } + } + (tmp_path / "journal" / "raw_events.jsonl").write_text( + json.dumps(journal) + "\n", encoding="utf-8" + ) + rows = [ + { + "id": "a1", "sequence": 1, "surface": "windows", "action_type": "click", + "target_snapshot": { + "name": "search_result", "control_type": "TreeControl", + "bounds": {"width": 600, "height": 500}, + }, + "evidence": {"eventual_observation_id": "obs"}, + }, + { + "id": "a2", "sequence": 2, "surface": "windows", "action_type": "press_key", + "target_snapshot": { + "name": "WeCom", "control_type": "WindowControl", + "bounds": {"width": 1440, "height": 900}, + }, + "input": {"key": "Enter"}, + "evidence": {"eventual_observation_id": "obs"}, + }, + ] + for row in rows: + (tmp_path / "actions" / f"{row['id']}.json").write_text( + json.dumps(row), encoding="utf-8" + ) + codes = {issue["code"] for issue in validate_capture(tmp_path)["issues"]} + assert "CLICK_TARGET_GENERIC_CONTAINER" in codes + assert "KEY_TARGET_ROOT_WINDOW" in codes + assert "TEXT_INPUT_TRANSACTION_MISSING" in codes + replay = build_replay_preflight(tmp_path) + assert replay["ready"] is False + assert replay["quality_gate"]["passed"] is False + assert replay["summary"]["quality_blockers"] >= 1 + + +def test_validator_flags_missing_production_locator_and_raw_link(tmp_path: Path) -> None: + import json + + actions = tmp_path / "actions" + elements = tmp_path / "elements" + actions.mkdir() + elements.mkdir() + (tmp_path / "observations").mkdir() + (tmp_path / "journal").mkdir() + (tmp_path / "journal" / "raw_events.jsonl").write_text("{}\n", encoding="utf-8") + (elements / "el.json").write_text( + json.dumps({"id": "el", "primary_locator": None}), encoding="utf-8" + ) + (actions / "a.json").write_text( + json.dumps({ + "id": "a", "sequence": 1, "surface": "browser", + "action_type": "click", "target_element_id": "el", + "target_snapshot": {"role": "link", "rect": {"width": 20, "height": 20}}, + "raw_event": {}, "evidence": {"eventual_observation_id": "obs"}, + }), + encoding="utf-8", + ) + result = validate_capture(tmp_path) + codes = {issue["code"] for issue in result["issues"]} + assert "ACTION_PRODUCTION_LOCATOR_MISSING" in codes + assert "ACTION_RAW_EVENT_LINK_MISSING" in codes + + +def test_validator_flags_graphic_leaf_low_stability_and_split_fill(tmp_path: Path) -> None: + import json + + actions = tmp_path / "actions" + elements = tmp_path / "elements" + actions.mkdir() + elements.mkdir() + (tmp_path / "observations").mkdir() + (tmp_path / "journal").mkdir() + (tmp_path / "journal" / "raw_events.jsonl").write_text("{}\n", encoding="utf-8") + locator = { + "strategy": "css", "expression": "css:.hash", "actual_count": 1, + "unique": True, "production_eligible": True, "stability_score": 0.32, + "validation": "live", + } + (elements / "el.json").write_text( + json.dumps({"id": "el", "primary_locator": locator}), encoding="utf-8" + ) + rows = [ + { + "id": "a1", "sequence": 1, "surface": "browser", "action_type": "fill", + "target_element_id": "el", "target_snapshot": {"tag": "input"}, + "input": {"text": "a"}, + "raw_event": {"meta": {"event_id": "doc:1", "source_ts_ms": 1000}}, + "evidence": {"eventual_observation_id": "obs", "locator_unique": True}, + }, + { + "id": "a2", "sequence": 2, "surface": "browser", "action_type": "fill", + "target_element_id": "el", "target_snapshot": {"tag": "input"}, + "input": {"text": "ai"}, + "raw_event": {"meta": {"event_id": "doc:2", "source_ts_ms": 1100}}, + "evidence": {"eventual_observation_id": "obs", "locator_unique": True}, + }, + { + "id": "a3", "sequence": 3, "surface": "browser", "action_type": "click", + "target_element_id": "el", "target_snapshot": {"tag": "path"}, + "raw_event": {"meta": {"event_id": "doc:3"}}, + "evidence": {"eventual_observation_id": "obs", "locator_unique": True}, + }, + ] + for row in rows: + (actions / f"{row['id']}.json").write_text(json.dumps(row), encoding="utf-8") + codes = {issue["code"] for issue in validate_capture(tmp_path)["issues"]} + assert "DUPLICATE_FILL_TRANSACTION" in codes + assert "GRAPHIC_LEAF_TARGET" in codes + assert "LOW_STABILITY_LOCATOR" in codes + + +def test_replay_preflight_is_honest_about_event_time_validation(tmp_path: Path) -> None: + import json + + (tmp_path / "actions").mkdir() + (tmp_path / "elements").mkdir() + (tmp_path / "actions" / "a.json").write_text(json.dumps({ + "id": "a", "sequence": 1, "action_type": "click", + "target_element_id": "el", "raw_event_ids": ["doc:1"], + "raw_event": {"meta": {"page_id": "page", "document_id": "doc"}}, + }), encoding="utf-8") + (tmp_path / "elements" / "el.json").write_text(json.dumps({ + "id": "el", "logical_name": "go", "primary_locator": { + "strategy": "role", "expression": "role:button,Go", + "actual_count": 1, "unique": True, "production_eligible": True, + "stability_score": 0.95, "validation": "live", + }, + }), encoding="utf-8") + report = build_replay_preflight(tmp_path) + assert report["mode"] == "static_non_mutating" + assert report["summary"]["pass"] == 1 + assert report["actions"][0]["verification"] == "event_time_unique" + + +def test_replay_preflight_rejects_unique_root_window_click(tmp_path: Path) -> None: + import json + + (tmp_path / "actions").mkdir() + (tmp_path / "elements").mkdir() + (tmp_path / "actions" / "a.json").write_text(json.dumps({ + "id": "a", "sequence": 1, "surface": "windows", "action_type": "click", + "target_element_id": "root", "target_snapshot": { + "control_type": "WindowControl", "name": "WeCom", + "bounds": {"width": 1440, "height": 900}, + }, + }), encoding="utf-8") + (tmp_path / "elements" / "root.json").write_text(json.dumps({ + "id": "root", "logical_name": "WeCom", "primary_locator": { + "strategy": "window_name_type", "expression": "window:WeCom", + "actual_count": 1, "unique": True, "production_eligible": True, + "stability_score": 0.9, "validation": "snapshot_only", + }, + }), encoding="utf-8") + report = build_replay_preflight(tmp_path) + assert report["summary"]["fail"] == 1 + assert report["actions"][0]["reason"] == "oversized_or_root_window_click_target" diff --git a/tools/ui_inspector/tests/test_locators_catalog.py b/tools/ui_inspector/tests/test_locators_catalog.py new file mode 100644 index 0000000..136974e --- /dev/null +++ b/tools/ui_inspector/tests/test_locators_catalog.py @@ -0,0 +1,215 @@ +"""Locator compiler / catalog / uniqueness tests.""" + +from __future__ import annotations + +import sys +import tempfile +import unittest +from pathlib import Path + +_HERE = Path(__file__).resolve().parents[1] +if str(_HERE) not in sys.path: + sys.path.insert(0, str(_HERE)) + +from locators.browser import compile_browser_locators +from locators.catalog import ElementCatalogService +from locators.validator import choose_production_primary, validate_locator_against_rows +from locators.windows import compile_windows_locators +from models.locator import LocatorCandidate + + +class TestBrowserLocators(unittest.TestCase): + def test_ordering_prefers_role_and_testid(self) -> None: + el = { + "role": "button", + "name": "Submit", + "test_id": "submit-btn", + "text": "Submit", + "id": "x1", + } + rows = [ + {"role": "button", "name": "Submit", "test_id": "submit-btn", "text": "Submit", "id": "x1"}, + {"role": "button", "name": "Cancel", "test_id": "cancel-btn", "text": "Cancel"}, + ] + cands = compile_browser_locators(el, rows=rows) + strategies = [c.strategy for c in cands] + self.assertIn("role", strategies) + self.assertIn("test_id", strategies) + primary = choose_production_primary(cands) + self.assertIsNotNone(primary) + assert primary is not None + self.assertFalse(primary.uses_coordinates) + + def test_contenteditable_ignores_dynamic_value_and_uses_role_css(self) -> None: + cands = compile_browser_locators( + { + "role": "combobox", + "name": "user typed comment", + "text": "user typed comment", + "value": "user typed comment", + "contenteditable": True, + "css": "#comment-input [contenteditable=true]", + } + ) + role = next(c for c in cands if c.strategy == "role") + css = next(c for c in cands if c.strategy == "css") + self.assertEqual(role.properties, {"role": "combobox"}) + self.assertNotIn("user typed comment", role.expression) + self.assertFalse(css.production_eligible) + + def test_live_unique_css_becomes_executable_fallback(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + service = ElementCatalogService(Path(tmp), flow_id="f") + el = service.upsert_from_snapshot( + { + "role": "path", + "tag": "path", + "css": "#comment-input .submit svg path", + "fingerprint": "path|submit", + }, + surface="browser", + frame={"url": "https://example.test/video"}, + live_counts={"css": 1}, + ) + self.assertIsNotNone(el.primary_locator) + assert el.primary_locator is not None + self.assertEqual(el.primary_locator.strategy, "css") + self.assertTrue(el.primary_locator.unique) + + def test_link_href_beats_ambiguous_card_css(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + service = ElementCatalogService(Path(tmp), flow_id="f") + el = service.upsert_from_snapshot( + { + "role": "link", + "tag": "a", + "name": "A result card with dynamic text", + "href": "/detail/42", + "href_count": 1, + "css": "#results > li.card > a.card-link", + "css_count": 24, + "fingerprint": "a|card", + }, + surface="browser", + frame={"url": "https://example.test/search"}, + live_counts={"href": 1, "css": 24}, + ) + self.assertIsNotNone(el.primary_locator) + assert el.primary_locator is not None + self.assertEqual(el.primary_locator.strategy, "href") + self.assertEqual(el.primary_locator.actual_count, 1) + self.assertTrue(el.primary_locator.production_eligible) + + def test_scoped_contenteditable_stable_css_can_be_primary(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + service = ElementCatalogService(Path(tmp), flow_id="f") + el = service.upsert_from_snapshot( + { + "role": "combobox", + "tag": "div", + "contenteditable": True, + "stable_css": "#comment-form div[contenteditable]", + "stable_css_count": 1, + "css": "#comment-form > div.hash-a > div.hash-b", + "css_count": 1, + "fingerprint": "div|editor", + }, + surface="browser", + frame={"url": "https://example.test/detail"}, + live_counts={"stable_css": 1, "css": 1}, + ) + self.assertIsNotNone(el.primary_locator) + assert el.primary_locator is not None + self.assertEqual(el.primary_locator.strategy, "stable_css") + self.assertGreater(el.primary_locator.stability_score, 0.7) + + +class TestWindowsLocators(unittest.TestCase): + def test_ordering_and_name_value_split(self) -> None: + el = { + "automation_id": "searchedit", + "name": "searchedit", + "value": "张三", + "control_type": "EditControl", + "parent_chain": [{"name": "search_layout"}], + } + rows = [ + { + "automation_id": "searchedit", + "name": "searchedit", + "value": "张三", + "control_type": "EditControl", + } + ] + cands = compile_windows_locators( + el, rows=rows, window_scope={"class_name": "WeWorkWindow", "title": "企业微信"} + ) + strategies = [c.strategy for c in cands] + self.assertTrue(strategies[0].startswith("window_automation_id")) + self.assertIn("window_name_value_type", strategies) + primary = choose_production_primary(cands) + self.assertIsNotNone(primary) + assert primary is not None + self.assertEqual(primary.actual_count, 1) + + def test_coordinates_never_primary(self) -> None: + el = { + "name": "x", + "control_type": "ButtonControl", + "bounds": {"left": 1, "top": 2, "right": 3, "bottom": 4}, + } + # Two matches → ambiguous for name+type; coords present + rows = [ + {"name": "x", "control_type": "ButtonControl"}, + {"name": "x", "control_type": "ButtonControl"}, + ] + cands = compile_windows_locators(el, rows=rows) + primary = choose_production_primary(cands) + self.assertIsNone(primary) + self.assertTrue(any(c.uses_coordinates for c in cands)) + for c in cands: + if c.uses_coordinates: + self.assertFalse(c.production_eligible) + + +class TestUniqueness(unittest.TestCase): + def test_ambiguous_not_primary(self) -> None: + c = LocatorCandidate( + strategy="name", + properties={"name": "item", "control_type": "ButtonControl"}, + stability_score=0.8, + ) + rows = [ + {"name": "item", "control_type": "ButtonControl"}, + {"name": "item", "control_type": "ButtonControl"}, + ] + validate_locator_against_rows(c, rows) + self.assertTrue(c.ambiguous) + self.assertFalse(c.production_eligible) + self.assertIsNone(choose_production_primary([c])) + + +class TestCatalogMerge(unittest.TestCase): + def test_merge_observations(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + svc = ElementCatalogService(Path(tmp), flow_id="f1") + el = { + "automation_id": "ok", + "name": "ok", + "control_type": "ButtonControl", + "value": "", + } + rows = [el] + a = svc.upsert_from_snapshot( + el, surface="windows", window_scope={"class_name": "X"}, rows=rows + ) + b = svc.upsert_from_snapshot( + el, surface="windows", window_scope={"class_name": "X"}, rows=rows + ) + self.assertEqual(a.id, b.id) + self.assertEqual(b.observation_count, 2) + self.assertTrue(svc.catalog_path.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/ui_inspector/tests/test_models_fingerprint_diff.py b/tools/ui_inspector/tests/test_models_fingerprint_diff.py new file mode 100644 index 0000000..dbf33d0 --- /dev/null +++ b/tools/ui_inspector/tests/test_models_fingerprint_diff.py @@ -0,0 +1,295 @@ +"""Phase 2 unit tests: models, fingerprint, dedupe, state diff, redaction.""" + +from __future__ import annotations + +import sys +import tempfile +import unittest +from pathlib import Path + +_HERE = Path(__file__).resolve().parents[1] +if str(_HERE) not in sys.path: + sys.path.insert(0, str(_HERE)) + +from models.action import ActionFlow, ActionRecord +from models.element import ElementRef, element_fingerprint +from models.locator import LocatorCandidate +from models.serialize import dumps_json, to_dict +from models.state import UIState, compute_state_fingerprint +from core.policies.element_identity import element_identity_key +from redaction import redact_input_payload, redact_text +from snapshot_service import SnapshotService +from state_diff import diff_states, propose_postconditions + + +class TestSerialization(unittest.TestCase): + def test_state_serialize(self) -> None: + st = UIState( + surface="browser", + driver="playwright", + url="https://example.com", + page_title="Example", + locatable=[{"control_type": "button", "name": "Go", "value": ""}], + ) + st.ensure_fingerprint() + raw = dumps_json(st) + self.assertIn("openclaw.ui_inspector.state.v1", raw) + self.assertIn("https://example.com", raw) + self.assertNotIn("\\u", raw) # ensure_ascii=False for CJK; ASCII ok either way + d = to_dict(st) + self.assertEqual(d["schema"], "openclaw.ui_inspector.state.v1") + self.assertTrue(d["fingerprint"]) + + def test_action_serialize(self) -> None: + act = ActionRecord( + action_type="click", + surface="windows", + before_state_id="st_a", + after_state_id="st_b", + sequence=1, + ) + d = to_dict(act) + self.assertEqual(d["schema"], "openclaw.ui_inspector.action.v1") + self.assertEqual(d["capability"], "implemented") + flow = ActionFlow(flow_id="f1", flow_name="demo", actions=[act.id]) + self.assertEqual(to_dict(flow)["schema"], "openclaw.ui_inspector.action_flow.v1") + + def test_element_locator_serialize(self) -> None: + loc = LocatorCandidate( + strategy="automation_id", + properties={"automation_id": "ok", "control_type": "Button"}, + expression="AutomationId=ok", + stability_score=0.9, + ) + loc.mark_counts(1) + el = ElementRef( + logical_name="ok_button", + surface="windows", + automation_id="ok", + control_type="ButtonControl", + locator_candidates=[loc], + primary_locator=loc, + ) + el.ensure_fingerprint() + d = to_dict(el) + self.assertEqual(d["schema"], "openclaw.ui_inspector.element.v1") + self.assertTrue(d["primary_locator"]["production_eligible"]) + + +class TestFingerprint(unittest.TestCase): + def test_stable_fingerprint(self) -> None: + kwargs = dict( + surface="browser", + url="https://a.test/x", + page_title="T", + window={"title": "T", "class_name": "Chrome", "process_name": "chrome.exe"}, + locatable=[ + {"control_type": "textbox", "name": "q", "value": "hello", "automation_id": "q"} + ], + ) + a = compute_state_fingerprint(**kwargs) + b = compute_state_fingerprint(**kwargs) + self.assertEqual(a, b) + + def test_session_fields_ignored(self) -> None: + base_loc = [{"control_type": "button", "name": "A", "value": "", "automation_id": "a"}] + fp1 = compute_state_fingerprint( + surface="windows", + window={"title": "App", "class_name": "X", "pid": 111, "hwnd": 222}, + locatable=base_loc, + ) + fp2 = compute_state_fingerprint( + surface="windows", + window={"title": "App", "class_name": "X", "pid": 999, "hwnd": 888}, + locatable=base_loc, + ) + self.assertEqual(fp1, fp2) + + def test_url_changes_fingerprint(self) -> None: + loc = [{"control_type": "button", "name": "A", "value": "", "automation_id": "a"}] + a = compute_state_fingerprint(surface="browser", url="https://a.test/1", locatable=loc) + b = compute_state_fingerprint(surface="browser", url="https://a.test/2", locatable=loc) + self.assertNotEqual(a, b) + + def test_anonymous_dom_elements_do_not_merge_across_paths(self) -> None: + common = dict(surface="browser", role="span", name="", value="") + a = element_fingerprint(**common, dom_fingerprint="span|#video-tab") + b = element_fingerprint(**common, dom_fingerprint="span|#comment-input") + self.assertNotEqual(a, b) + + def test_contenteditable_value_is_not_identity(self) -> None: + common = dict( + surface="browser", + role="combobox", + contenteditable=True, + dom_fingerprint="div|#comment-input [contenteditable]", + ) + self.assertEqual( + element_fingerprint(**common, name="first", value="first"), + element_fingerprint(**common, name="second", value="second"), + ) + + def test_link_href_is_identity_when_visible_text_changes(self) -> None: + common = { + "role": "link", + "tag": "a", + "href": "/detail/42", + "css": "#results > li.card > a", + } + self.assertEqual( + element_identity_key( + {**common, "name": "Result with 10 likes"}, + surface="browser", + page_url="https://example.test/search?page=1", + ), + element_identity_key( + {**common, "name": "Result with 11 likes"}, + surface="browser", + page_url="https://example.test/search?page=2", + ), + ) + + def test_value_change_detected(self) -> None: + a = compute_state_fingerprint( + surface="windows", + locatable=[{"control_type": "Edit", "name": "box", "value": "1", "automation_id": "e"}], + ) + b = compute_state_fingerprint( + surface="windows", + locatable=[{"control_type": "Edit", "name": "box", "value": "2", "automation_id": "e"}], + ) + self.assertNotEqual(a, b) + + +class TestDedupe(unittest.TestCase): + def test_same_state_deduped(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + svc = SnapshotService(Path(tmp), persist_evidence=False) + + class Dump: + surface = "browser" + driver = "playwright" + window = {"title": "T", "class_name": "P"} + tree = {} + index = [] + locatable = [ + {"control_type": "button", "name": "Go", "value": "", "automation_id": "go"} + ] + stats = {"locatable_count": 1, "nodes_kept": 1} + selection = {} + url = "https://example.com" + page_title = "T" + screenshot_bytes = None + use_hwnd_screenshot = False + + s1 = svc.capture_from_dump(Dump(), change_reason="entry") + s2 = svc.capture_from_dump(Dump(), change_reason="heartbeat") + self.assertEqual(s1.id, s2.id) + self.assertEqual(s2.observation_count, 2) + self.assertEqual(len(svc.state_ids()), 1) + + +class TestStateDiff(unittest.TestCase): + def test_diff_and_postconditions(self) -> None: + before = { + "url": "https://a.test/1", + "page_title": "One", + "window": {"class_name": "WeWorkWindow", "title": "企业微信", "hwnd": 1}, + "locatable": [ + { + "control_type": "Edit", + "name": "searchedit", + "value": "q", + "automation_id": "searchedit", + } + ], + "stats": {"locatable_count": 1}, + } + after = { + "url": "https://a.test/1", + "page_title": "Chat", + "window": {"class_name": "WeWorkWindow", "title": "企业微信", "hwnd": 1}, + "locatable": [ + { + "control_type": "Edit", + "name": "send_rich_edit", + "value": "", + "automation_id": "send_rich_edit", + } + ], + "stats": {"locatable_count": 1}, + } + changes = diff_states(before, after) + kinds = {c.kind for c in changes} + self.assertIn("title_changed", kinds) + self.assertIn("element_appeared", kinds) + self.assertIn("element_disappeared", kinds) + props = propose_postconditions(changes) + self.assertTrue(all(p.status == "proposed" for p in props)) + self.assertTrue(any("appeared" in p.description.lower() or "出现" in p.description or "appeared" in p.description for p in props) or props) + + +class TestRedaction(unittest.TestCase): + def test_password_redacted(self) -> None: + out, sens = redact_input_payload( + {"text": "secret123", "name": "password"}, + field_name="password", + input_type="password", + ) + self.assertTrue(sens) + self.assertEqual(out["text"], "[REDACTED]") + + def test_phone_redacted(self) -> None: + self.assertIn("[REDACTED]", redact_text("call 13800138000 please")) + + def test_raw_tree_and_index_are_sanitized_before_write(self) -> None: + secret = "Bearer_abcdefghijklmnopqrstuvwxyz123456" + + class SensitiveDump: + surface = "browser" + driver = "playwright" + window = {"title": "Login"} + tree = { + "role": "textbox", + "name": "password", + "input_type": "password", + "value": secret, + } + index = [dict(tree)] + locatable = [dict(tree)] + stats = {} + selection = {} + url = "https://example.com/login" + page_title = "Login" + screenshot_bytes = None + use_hwnd_screenshot = False + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + svc = SnapshotService(root) + svc.capture_from_dump( + SensitiveDump(), change_reason="sensitive", force_persist=True + ) + persisted = "\n".join( + p.read_text(encoding="utf-8", errors="ignore") + for p in root.rglob("*") + if p.is_file() + ) + self.assertNotIn(secret, persisted) + + +class TestActionBeforeAfter(unittest.TestCase): + def test_click_links_states(self) -> None: + act = ActionRecord( + action_type="click", + before_state_id="st_before", + after_state_id="st_after", + target_snapshot={"name": "btn"}, + ) + self.assertEqual(act.before_state_id, "st_before") + self.assertEqual(act.after_state_id, "st_after") + self.assertEqual(act.capability, "implemented") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/ui_inspector/tests/test_recorder_adapters.py b/tools/ui_inspector/tests/test_recorder_adapters.py new file mode 100644 index 0000000..23fe6c3 --- /dev/null +++ b/tools/ui_inspector/tests/test_recorder_adapters.py @@ -0,0 +1,1513 @@ +"""Recorder / aggregator / adapter / heartbeat tests.""" + +from __future__ import annotations + +import sys +import tempfile +import threading +import time +import unittest +from pathlib import Path +from unittest.mock import patch + +_HERE = Path(__file__).resolve().parents[1] +if str(_HERE) not in sys.path: + sys.path.insert(0, str(_HERE)) + +from adapters.wecom import wecom_adapter +from flow_session import FlowSession +from evidence.coordinator import EvidenceCoordinator +import kit_browser +from kit_wake import ( + _cache_key, + _caches, + auto_ensure_if_wecom, + note_provider_warm, + resolve_wake_profile, +) +from models.action import ActionRecord +from recorder.base import RawUserEvent, RecorderState +from recorder.browser import BrowserRecorder +from recorder.input_aggregator import InputAggregator +from recorder.manager import ActionRecordingManager +from recorder.windows import ( + WindowsRecorder, + _WM_LBUTTONDOWN, + _WM_LBUTTONUP, +) +from redaction import redact_element_fields, sanitize_artifact +from snapshot_service import SnapshotService + + +class Dump: + def __init__(self, url="https://example.com", value="x", title="T"): + self.surface = "browser" + self.driver = "playwright" + self.window = {"title": title, "class_name": "P"} + self.tree = {} + self.index = [] + self.locatable = [ + { + "control_type": "textbox", + "name": "q", + "value": value, + "automation_id": "q", + } + ] + self.stats = {"locatable_count": 1, "nodes_kept": 1} + self.selection = {} + self.url = url + self.page_title = title + self.screenshot_bytes = None + self.use_hwnd_screenshot = False + + +class TestInputAggregator(unittest.TestCase): + def test_aggregates_chars(self) -> None: + agg = InputAggregator(idle_sec=0.5) + out = [] + out += agg.on_key(key="a", text="a", target={"name": "q", "fingerprint": "1"}) + out += agg.on_key(key="b", text="ab", target={"name": "q", "fingerprint": "1"}) + self.assertEqual(out, []) + flushed = agg.flush() + self.assertEqual(len(flushed), 1) + self.assertEqual(flushed[0].action_type, "fill") + self.assertEqual(flushed[0].text, "ab") + + def test_enter_is_press_key(self) -> None: + agg = InputAggregator() + items = agg.on_key(key="Enter", target={"name": "q"}) + self.assertEqual(len(items), 1) + self.assertEqual(items[0].action_type, "press_key") + self.assertEqual(items[0].key, "Enter") + + def test_sensitive_fill(self) -> None: + agg = InputAggregator() + agg.on_key( + key="x", + text="secret", + target={"name": "password", "fingerprint": "p"}, + input_type="password", + field_name="password", + ) + items = agg.flush() + self.assertTrue(items[0].sensitive) + self.assertEqual(items[0].input_payload.get("text"), "[REDACTED]") + + def test_empty_value_change_records_clear_as_fill(self) -> None: + agg = InputAggregator() + agg.on_value_change( + value="", + target={"name": "comment", "fingerprint": "editor"}, + capture_context={"page_id": "page_detail"}, + ) + items = agg.flush() + self.assertEqual(len(items), 1) + self.assertEqual(items[0].action_type, "fill") + self.assertEqual(items[0].input_payload.get("text"), "") + self.assertEqual(items[0].capture_context.get("page_id"), "page_detail") + + def test_focus_class_change_stays_one_fill_transaction(self) -> None: + agg = InputAggregator(idle_sec=8.0) + context = { + "surface": "browser", + "url": "https://example.test/search", + "frame_name": "", + } + first = { + "role": "textbox", "tag": "input", "placeholder": "Search", + "class_name": "field", "fingerprint": "input|field", + "stable_css": '#header input[placeholder="Search"]', + } + focused = { + **first, + "class_name": "field focused", + "fingerprint": "input|field.focused", + } + out = agg.on_value_change( + value="a", target=first, capture_context=context, now=1000.0, + raw_event={"meta": {"event_id": "doc:9"}}, + ) + out += agg.on_value_change( + value="ai", target=focused, capture_context=context, now=1000.136, + raw_event={"meta": {"event_id": "doc:10"}}, + ) + self.assertEqual(out, []) + flushed = agg.flush(now=1000.2) + self.assertEqual(len(flushed), 1) + self.assertEqual(flushed[0].input_payload["text"], "ai") + self.assertEqual(flushed[0].raw_event_ids, ["doc:9", "doc:10"]) + + +class TestWindowsRecorder(unittest.TestCase): + def test_machine_element_name_is_not_redacted_as_a_secret(self) -> None: + cleaned, sensitive = redact_element_fields( + {"name": "OperationAndSendMessageRegion", "control_type": "PaneControl"} + ) + self.assertEqual(cleaned["name"], "OperationAndSendMessageRegion") + self.assertFalse(sensitive) + + def test_shadow_windows_are_decorative_not_business_overlays(self) -> None: + self.assertTrue(WindowsRecorder._is_decorative_window_class("PerryShadowWnd")) + self.assertTrue(WindowsRecorder._is_decorative_window_class("AcmeDropShadow")) + self.assertFalse(WindowsRecorder._is_decorative_window_class("SearchResultWindow2")) + + def test_persisted_atlas_prefers_exact_editor_over_layout(self) -> None: + recorder = WindowsRecorder() + bounds = { + "left": 336, "top": 660, "right": 1422, "bottom": 798, + "width": 1086, "height": 138, + } + recorder.learn_elements( + [ + {"name": "send_rich_edit_layout", "control_type": "TreeControl", "rect": bounds}, + {"name": "send_rich_edit", "control_type": "TreeControl", "rect": bounds}, + ], + window={"class_name": "WeWorkWindow", "hwnd": 10, "pid": 20}, + ) + hit = recorder._atlas_hit("WeWorkWindow", [(439, 673)]) + self.assertEqual(hit.get("name"), "send_rich_edit") + self.assertEqual(hit.get("value"), "") + + def test_typing_activity_survives_temporarily_unresolved_target(self) -> None: + seen = [] + recorder = WindowsRecorder(on_event=seen.append) + recorder.status = RecorderState.RECORDING + recorder._last_root_hwnd = 10 + recorder._last_uia_point = (100, 200) + recorder._emit_typing_activity({}, {"source_time_ms": 1}) + self.assertEqual([event.kind for event in seen], ["typing_activity"]) + self.assertTrue(seen[0].meta.get("unresolved_target")) + self.assertEqual(seen[0].meta.get("root_hwnd"), 10) + + def test_input_method_candidate_window_is_not_a_business_target(self) -> None: + self.assertTrue( + WindowsRecorder._is_transient_input_target( + { + "control_type": "PaneControl", + "class_name": "SoPY_Comp", + "window_scope": {"class_name": "SoPY_Comp"}, + } + ) + ) + self.assertFalse( + WindowsRecorder._is_transient_input_target( + {"name": "send_rich_edit", "control_type": "TreeControl"} + ) + ) + + def test_backspace_in_editor_is_folded_into_value_transaction(self) -> None: + seen = [] + recorder = WindowsRecorder(on_event=seen.append) + recorder.status = RecorderState.RECORDING + editor = { + "name": "send_rich_edit", "control_type": "TreeControl", + "window_scope": {"hwnd": 10}, + } + recorder._publish_focus_snapshot(editor) + recorder._on_key_down({"vk": 0x08, "source_time_ms": 1}) + self.assertEqual([event.kind for event in seen], ["typing_activity"]) + self.assertNotIn("Backspace", [event.key for event in seen]) + + def test_ime_enter_is_not_emitted_as_business_key(self) -> None: + seen = [] + recorder = WindowsRecorder(on_event=seen.append) + recorder.status = RecorderState.RECORDING + editor = { + "name": "send_rich_edit", "control_type": "TreeControl", + "window_scope": {"hwnd": 10, "pid": 20, "class_name": "WeWorkWindow"}, + } + recorder._last_text_entry_target = dict(editor) + + recorder._ime_composing = True + recorder._publish_focus_snapshot(editor) + recorder._on_key_down({"vk": 0x0D, "source_time_ms": 1}) + self.assertEqual([event.kind for event in seen], ["typing_activity"]) + self.assertNotIn("Enter", [event.key for event in seen]) + + def test_key_consumer_never_calls_uia(self) -> None: + seen = [] + recorder = WindowsRecorder(on_event=seen.append) + recorder.status = RecorderState.RECORDING + recorder._publish_focus_snapshot( + {"name": "send_rich_edit", "control_type": "TreeControl", "window_scope": {"hwnd": 10}} + ) + with patch.object( + recorder, "_focused_control_snapshot", side_effect=AssertionError("UIA on input thread") + ), patch.object( + recorder, "_poll_focus_value", side_effect=AssertionError("UIA on input thread") + ): + for _ in range(100): + recorder._on_key_down({"vk": 0x41, "source_time_ms": 1}) + self.assertEqual([event.kind for event in seen], ["typing_activity"]) + + def test_slow_uia_sampler_does_not_block_burst_input_consumer(self) -> None: + recorder = WindowsRecorder() + recorder.status = RecorderState.RECORDING + recorder._stop.clear() + recorder._uia_finished.clear() + slow_started = threading.Event() + + def slow_snapshot(): + slow_started.set() + time.sleep(0.45) + return {} + + with patch.object(recorder, "_focused_control_snapshot", side_effect=slow_snapshot): + consumer = threading.Thread(target=recorder._capture_loop) + sampler = threading.Thread(target=recorder._uia_sampler_loop) + consumer.start() + sampler.start() + recorder._uia_request.set() + self.assertTrue(slow_started.wait(0.2)) + for index in range(120): + recorder._enqueue_input({"kind": "key", "vk": 0x41, "source_time_ms": index}) + deadline = time.monotonic() + 0.3 + while recorder.health()["resolved_events"] < 120 and time.monotonic() < deadline: + time.sleep(0.005) + self.assertEqual(recorder.health()["resolved_events"], 120) + self.assertEqual(recorder._input_queue.qsize(), 0) + recorder._stop.set() + recorder._uia_request.set() + sampler.join(1.5) + consumer.join(1.5) + self.assertFalse(sampler.is_alive()) + self.assertFalse(consumer.is_alive()) + + def test_stop_drains_every_accepted_input_packet(self) -> None: + recorder = WindowsRecorder() + recorder.status = RecorderState.RECORDING + recorder._stop.clear() + recorder._uia_finished.clear() + with patch.object(recorder, "_focused_control_snapshot", return_value={}): + recorder._capture_thread = threading.Thread(target=recorder._capture_loop) + recorder._uia_thread = threading.Thread(target=recorder._uia_sampler_loop) + recorder._capture_thread.start() + recorder._uia_thread.start() + for index in range(300): + recorder._enqueue_input({"kind": "key", "vk": 0x41, "source_time_ms": index}) + recorder.stop() + health = recorder.health() + self.assertEqual(health["resolved_events"], 300) + self.assertEqual(health["undrained_events"], 0) + self.assertTrue(health["shutdown_complete"]) + + def test_value_is_flushed_before_enter_boundary(self) -> None: + seen = [] + recorder = WindowsRecorder(on_event=seen.append) + recorder.status = RecorderState.RECORDING + editor = { + "name": "send_rich_edit", "control_type": "TreeControl", "value": "hello", + "window_scope": {"hwnd": 10}, + } + recorder._last_focus_fp = recorder._focus_fingerprint(editor) + recorder._last_focus_value = "" + recorder._typing_active = True + recorder._publish_focus_snapshot(editor) + recorder._on_key_down({"vk": 0x0D, "source_time_ms": 2}) + self.assertEqual([event.kind for event in seen], ["value_change", "key"]) + self.assertEqual(seen[0].text, "hello") + + def test_focused_control_reprobes_mixed_dpi_point_after_provider_warms(self) -> None: + recorder = WindowsRecorder() + recorder._last_raw_point = (784, 1358) + recorder._last_uia_point = (782, 1357) + recorder._last_root_hwnd = 10 + root = { + "name": "WeCom", "control_type": "WindowControl", + "bounds": {"width": 2880, "height": 1800}, + "window_scope": {"hwnd": 10, "pid": 20, "class_name": "WeWorkWindow"}, + } + editor = { + "name": "send_rich_edit", "control_type": "TreeControl", + "value": "hello", "bounds": {"left": 336, "top": 679, "right": 1060, "bottom": 817, "width": 724, "height": 138}, + "window_scope": {"hwnd": 10, "pid": 20, "class_name": "WeWorkWindow"}, + } + fake_auto = type( + "FakeAuto", + (), + { + "GetFocusedControl": staticmethod(lambda: object()), + "ControlFromPoint": staticmethod(lambda *_args: object()), + }, + ) + with patch.dict(sys.modules, {"uiautomation": fake_auto}), patch.object( + recorder, "_control_snapshot", return_value=root + ), patch.object( + recorder, + "_probe_mixed_uia_coordinates", + return_value=(editor, 392, 679, {"coordinate_selection": "window_dpi_absolute"}), + ): + focused = recorder._focused_control_snapshot() + self.assertEqual(focused.get("name"), "send_rich_edit") + + def test_transient_focus_falls_back_to_last_real_editor(self) -> None: + recorder = WindowsRecorder() + editor = { + "name": "searchedit", "control_type": "TreeControl", + "window_scope": {"hwnd": 10, "pid": 20, "class_name": "WeWorkWindow"}, + } + transient = { + "class_name": "SoPY_Comp", "control_type": "PaneControl", + "bounds": {"width": 400, "height": 100}, + "window_scope": {"hwnd": 99, "pid": 20, "class_name": "SoPY_Comp"}, + } + recorder._last_text_entry_target = dict(editor) + fake_auto = type( + "FakeAuto", (), {"GetFocusedControl": staticmethod(lambda: object())} + ) + with patch.dict(sys.modules, {"uiautomation": fake_auto}), patch.object( + recorder, "_control_snapshot", return_value=transient + ): + focused = recorder._focused_control_snapshot() + self.assertEqual(focused.get("name"), "searchedit") + self.assertTrue(recorder._ime_composing) + + def test_focus_value_change_emits_committed_text(self) -> None: + seen = [] + recorder = WindowsRecorder(on_event=seen.append) + recorder.status = RecorderState.RECORDING + recorder._last_focus_fp = "|send_rich_edit|TreeControl" + recorder._last_focus_value = "" + editor = { + "name": "send_rich_edit", "control_type": "TreeControl", + "value": "您好", "window_scope": {"hwnd": 10}, + } + with patch.object(recorder, "_focused_control_snapshot", return_value=editor): + recorder._poll_focus_value() + self.assertEqual(len(seen), 1) + self.assertEqual(seen[0].kind, "value_change") + self.assertEqual(seen[0].text, "您好") + + def test_typing_activity_is_coalesced_and_contains_no_key_text(self) -> None: + seen = [] + recorder = WindowsRecorder(on_event=seen.append) + recorder.status = RecorderState.RECORDING + editor = { + "name": "searchedit", "control_type": "TreeControl", + "window_scope": {"hwnd": 10}, + } + recorder._emit_typing_activity(editor, {"source_time_ms": 1}) + recorder._emit_typing_activity(editor, {"source_time_ms": 2}) + self.assertEqual(len(seen), 1) + self.assertEqual(seen[0].kind, "typing_activity") + self.assertEqual(seen[0].text, "") + self.assertFalse(seen[0].meta["content_recorded"]) + + def test_wecom_root_hit_probes_window_dpi_coordinate_candidate(self) -> None: + recorder = WindowsRecorder() + root = { + "name": "WeCom", + "control_type": "WindowControl", + "class_name": "WeWorkWindow", + "bounds": {"width": 2880, "height": 1800}, + "window_scope": { + "hwnd": 10, "pid": 20, "class_name": "WeWorkWindow", + }, + } + search = { + "name": "searchedit", + "control_type": "TreeControl", + "bounds": {"width": 139, "height": 22}, + "window_scope": { + "hwnd": 10, "pid": 20, "class_name": "WeWorkWindow", + }, + } + with patch( + "recorder.windows.ctypes.windll.user32.GetDpiForWindow", + return_value=192, + ), patch.object(recorder, "_element_from_point", return_value=(search, "")): + target, x, y, meta = recorder._probe_mixed_uia_coordinates( + 10, 282, 76, current_point=(280, 72), current_target=root + ) + self.assertEqual((x, y), (141, 38)) + self.assertEqual(target.get("name"), "searchedit") + self.assertEqual(meta.get("coordinate_selection"), "window_dpi_absolute") + + def test_physical_hook_point_prefers_actual_uia_root_coordinate_space(self) -> None: + class Rect: + left, top, right, bottom = 0, 0, 1440, 900 + + class Root: + BoundingRectangle = Rect() + + fake_auto = type("FakeAuto", (), {"ControlFromHandle": staticmethod(lambda _hwnd: Root())}) + + def physical_rect(_hwnd, _attr, ptr, _size): + rect = ptr._obj + rect.left, rect.top, rect.right, rect.bottom = 0, 0, 2880, 1800 + return 0 + + with patch.dict(sys.modules, {"uiautomation": fake_auto}), patch( + "recorder.windows.ctypes.windll.dwmapi.DwmGetWindowAttribute", + side_effect=physical_rect, + ): + x, y, meta = WindowsRecorder._uia_point_for_window( + 123, 884, 1396, coordinate_space="physical_screen" + ) + self.assertEqual((x, y), (442, 698)) + self.assertEqual(meta["coordinate_conversion"], "dwm_physical_to_uia_root") + self.assertEqual(meta["dpi_scale"], {"x": 2.0, "y": 2.0}) + + def test_physical_hook_point_is_mapped_to_dpi_virtualized_uia_space(self) -> None: + def logical_rect(_hwnd, ptr): + rect = ptr._obj + rect.left, rect.top, rect.right, rect.bottom = -2, -2, 1442, 854 + return 1 + + def physical_rect(_hwnd, _attr, ptr, _size): + rect = ptr._obj + rect.left, rect.top, rect.right, rect.bottom = 0, 0, 2880, 1708 + return 0 + + with patch( + "recorder.windows.ctypes.windll.user32.GetWindowRect", + side_effect=logical_rect, + ), patch( + "recorder.windows.ctypes.windll.dwmapi.DwmGetWindowAttribute", + side_effect=physical_rect, + ): + x, y, meta = WindowsRecorder._uia_point_for_window( + 123, 363, 90, coordinate_space="physical_screen" + ) + self.assertTrue(175 <= x <= 185) + self.assertTrue(40 <= y <= 47) + self.assertAlmostEqual(meta["dpi_scale"]["x"], 2.0, delta=0.02) + + def test_mouse_packets_preserve_hook_coordinates_and_injected_flag(self) -> None: + seen = [] + recorder = WindowsRecorder(on_event=seen.append) + recorder.status = RecorderState.RECORDING + with patch.object( + recorder, + "_element_from_point", + return_value=({"name": "Send", "control_type": "ButtonControl"}, ""), + ) as element_from_point, patch.object(recorder, "exclude_hwnds", return_value=set()), patch( + "recorder.windows.ctypes.windll.user32.WindowFromPoint", return_value=1 + ), patch( + "recorder.windows.ctypes.windll.user32.GetAncestor", return_value=1 + ): + recorder._health_set(backend="low_level_hooks") + recorder._dispatch_input_packet( + { + "kind": "mouse", + "message": _WM_LBUTTONDOWN, + "x": 321, + "y": 654, + "injected": True, + "source_time_ms": 1234, + } + ) + recorder._dispatch_input_packet( + {"kind": "mouse", "message": _WM_LBUTTONUP} + ) + self.assertEqual(len(seen), 1) + self.assertEqual((seen[0].x, seen[0].y), (321, 654)) + self.assertTrue(seen[0].meta["injected"]) + self.assertEqual(seen[0].meta["input_backend"], "low_level_hooks") + element_from_point.assert_not_called() + + def test_health_exposes_backend_and_queue_counters(self) -> None: + recorder = WindowsRecorder() + recorder._health_set(backend="low_level_hooks", mouse_hook=True) + recorder._enqueue_input({"kind": "mouse", "message": _WM_LBUTTONUP}) + health = recorder.health() + self.assertEqual(health["backend"], "low_level_hooks") + self.assertTrue(health["mouse_hook"]) + self.assertEqual(health["raw_input_events"], 1) + self.assertEqual(health["mouse_events"], 1) + + def test_all_inspector_process_windows_are_excluded(self) -> None: + seen = [] + recorder = WindowsRecorder(on_event=seen.append) + recorder.status = RecorderState.RECORDING + + def set_current_pid(_hwnd, pid_ptr): + import os + pid_ptr._obj.value = os.getpid() + return 1 + + with patch( + "recorder.windows.ctypes.windll.user32.WindowFromPoint", return_value=10 + ), patch( + "recorder.windows.ctypes.windll.user32.GetAncestor", return_value=20 + ), patch( + "recorder.windows.ctypes.windll.user32.GetWindowThreadProcessId", + side_effect=set_current_pid, + ), patch.object(recorder, "_element_from_point") as element_from_point: + recorder._on_mouse_down_at(100, 200) + self.assertEqual(seen, []) + element_from_point.assert_not_called() + + +class TestActionManager(unittest.TestCase): + def test_enter_inherits_the_fill_target_instead_of_root_window(self) -> None: + editor = { + "name": "send_rich_edit", "control_type": "TreeControl", + "bounds": {"width": 500, "height": 100}, + "window_scope": {"hwnd": 10, "pid": 20, "class_name": "WeWorkWindow"}, + } + root = { + "name": "WeCom", "control_type": "WindowControl", + "bounds": {"width": 1440, "height": 900}, + "window_scope": {"hwnd": 10, "pid": 20, "class_name": "WeWorkWindow"}, + } + with tempfile.TemporaryDirectory() as tmp: + mgr = ActionRecordingManager(Path(tmp), surface="windows", dump_fn=None) + mgr.start() + mgr.handle_raw_event( + RawUserEvent(kind="value_change", surface="windows", text="您好", target=editor) + ) + key = mgr.handle_raw_event( + RawUserEvent(kind="key", surface="windows", key="Enter", target=root) + ) + actions = sorted(mgr._actions_by_id.values(), key=lambda action: action.sequence) + self.assertEqual([action.action_type for action in actions], ["fill", "press_key"]) + self.assertEqual(key.target_snapshot.get("name"), "send_rich_edit") + + def test_unique_result_row_is_preferred_over_generic_search_container(self) -> None: + chosen = ActionRecordingManager._unique_windows_row_candidate( + [ + { + "name": "search_result", "control_type": "TreeControl", + "rect": {"left": 0, "top": 0, "right": 500, "bottom": 500, "width": 500, "height": 500}, + }, + { + "depth": 8, "name": "user_item", "control_type": "TreeControl", + "value": "", "rect": {"left": 0, "top": 50, "right": 500, "bottom": 110, "width": 500, "height": 60}, + }, + { + "depth": 9, "name": "search_name", "control_type": "TreeItemControl", + "value": "陈德炼", "rect": {"left": 60, "top": 60, "right": 180, "bottom": 90, "width": 120, "height": 30}, + }, + ] + ) + self.assertIsNotNone(chosen) + self.assertEqual(chosen.get("name"), "user_item") + self.assertEqual(chosen.get("value"), "陈德炼") + self.assertEqual(chosen.get("semantic_name"), "陈德炼") + + def test_unique_row_prefers_named_value_and_strips_rich_text_markup(self) -> None: + chosen = ActionRecordingManager._unique_windows_row_candidate( + [ + { + "depth": 8, "name": "user_item", "control_type": "TreeControl", + "rect": {"left": 0, "top": 50, "right": 500, "bottom": 110, "width": 500, "height": 60}, + }, + { + "depth": 9, "name": "search_avatar_label", "value": "avatar", + "control_type": "TreeItemControl", + }, + { + "depth": 9, "name": "search_name", + "value": "Alice@WeCom", + "control_type": "TreeItemControl", + }, + ] + ) + self.assertIsNotNone(chosen) + self.assertEqual(chosen.get("semantic_name"), "Alice@WeCom") + self.assertEqual(chosen.get("value"), "Alice@WeCom") + + def test_named_editor_beats_smaller_generic_list_item_at_same_point(self) -> None: + chosen = ActionRecordingManager._windows_hit_candidate( + [ + { + "name": "listitem_content", "control_type": "ListItemControl", + "rect": {"left": 100, "top": 500, "right": 300, "bottom": 560, "width": 200, "height": 60}, + }, + { + "name": "send_rich_edit", "control_type": "TreeControl", + "rect": {"left": 100, "top": 500, "right": 900, "bottom": 700, "width": 800, "height": 200}, + }, + ], + 150, + 530, + ) + self.assertEqual(chosen.get("name"), "send_rich_edit") + + def test_coordinate_promotion_cleans_search_result_rich_text(self) -> None: + class PopupDump(Dump): + def __init__(self): + super().__init__() + self.surface = "windows" + self.window = { + "hwnd": 30, "pid": 20, "class_name": "SearchResultWindow2" + } + self.locatable = [ + { + "name": "search_name", "control_type": "TreeItemControl", + "value": "Alice@WeCom", + "rect": {"left": 0, "top": 0, "right": 300, "bottom": 60, "width": 300, "height": 60}, + } + ] + + broad = { + "name": "SearchResultWindow2", "control_type": "PaneControl", + "bounds": {"left": 0, "top": 0, "right": 600, "bottom": 600, "width": 600, "height": 600}, + "window_scope": {"hwnd": 30, "pid": 20, "class_name": "SearchResultWindow2"}, + } + with tempfile.TemporaryDirectory() as tmp: + mgr = ActionRecordingManager(Path(tmp), surface="windows", dump_fn=None, defer_action_evidence=True) + mgr.start(entry_dump=PopupDump()) + action = mgr.handle_raw_event( + RawUserEvent(kind="click", surface="windows", x=20, y=20, target=broad) + ) + assert action is not None + mgr.attach_eventual_evidence( + [action.id], dump=PopupDump(), + requested_monotonic=__import__("time").monotonic(), + ) + self.assertEqual(action.target_snapshot.get("value"), "Alice@WeCom") + + def test_machine_identifier_is_not_redacted_as_token(self) -> None: + cleaned = sanitize_artifact( + {"name": "conversation_list_container", "value": ""} + ) + self.assertEqual(cleaned.get("name"), "conversation_list_container") + + def test_action_observer_receives_capture_and_evidence_phases(self) -> None: + phases = [] + with tempfile.TemporaryDirectory() as tmp: + mgr = ActionRecordingManager( + Path(tmp), surface="browser", dump_fn=None, + defer_action_evidence=True, + on_action_update=lambda action, phase: phases.append((action.id, phase)), + ) + mgr.start(entry_dump=Dump(value="entry")) + action = mgr.handle_raw_event( + RawUserEvent( + kind="click", surface="browser", + target={"role": "button", "name": "Go", "css_count": 1}, + ) + ) + assert action is not None + mgr.attach_eventual_evidence( + [action.id], dump=Dump(value="after"), + requested_monotonic=__import__("time").monotonic(), + ) + self.assertEqual([phase for _action_id, phase in phases], ["captured", "evidence"]) + + def test_windows_hit_candidate_rejects_one_pixel_separator(self) -> None: + chosen = ActionRecordingManager._windows_hit_candidate( + [ + { + "control_type": "TreeItemControl", + "name": "conv_list_sep_line", + "rect": {"left": 0, "top": 10, "right": 300, "bottom": 11, "width": 300, "height": 1}, + }, + { + "control_type": "TreeItemControl", + "name": "groupbuddyname", + "rect": {"left": 0, "top": 0, "right": 300, "bottom": 60, "width": 300, "height": 60}, + }, + ], + 90, + 10, + ) + self.assertIsNotNone(chosen) + self.assertEqual(chosen.get("name"), "groupbuddyname") + + def test_eventual_dump_never_promotes_action_from_another_window(self) -> None: + class WindowsDump(Dump): + def __init__(self): + super().__init__() + self.surface = "windows" + self.window = { + "hwnd": 10, "pid": 20, + "title": "WeCom", "class_name": "WeWorkWindow", + } + self.locatable = [ + { + "control_type": "TreeControl", "name": "searchedit", + "rect": {"left": 0, "top": 0, "right": 300, "bottom": 80, "width": 300, "height": 80}, + } + ] + + with tempfile.TemporaryDirectory() as tmp: + mgr = ActionRecordingManager( + Path(tmp), surface="windows", dump_fn=None, + defer_action_evidence=True, + ) + mgr.start(entry_dump=WindowsDump()) + action = mgr.handle_raw_event( + RawUserEvent( + kind="click", surface="windows", x=20, y=20, + target={ + "control_type": "GroupControl", "class_name": "QWidget", + "bounds": {"left": 0, "top": 0, "right": 2880, "bottom": 1800, "width": 2880, "height": 1800}, + "window_scope": {"hwnd": 99, "pid": 30, "class_name": "Qt51514QWindowIcon"}, + }, + meta={"uia_coordinates": {"x": 20, "y": 20}}, + ) + ) + assert action is not None + mgr.attach_eventual_evidence( + [action.id], dump=WindowsDump(), + requested_monotonic=__import__("time").monotonic(), + ) + self.assertEqual(action.target_snapshot.get("class_name"), "QWidget") + self.assertNotIn("target_promotion", action.evidence) + + def test_eventual_windows_tree_promotes_root_click_to_smallest_target(self) -> None: + class WindowsDump(Dump): + def __init__(self): + super().__init__(value="after") + self.surface = "windows" + self.driver = "uiautomation" + self.window = { + "hwnd": 10, "pid": 20, + "title": "WeCom", "class_name": "WeWorkWindow", + } + self.locatable = [ + { + "control_type": "WindowControl", "name": "WeCom", + "class_name": "WeWorkWindow", + "bounds": {"left": 0, "top": 0, "right": 1440, "bottom": 900, "width": 1440, "height": 900}, + }, + { + "control_type": "TreeControl", "name": "searchedit", + "bounds": {"left": 110, "top": 25, "right": 249, "bottom": 47, "width": 139, "height": 22}, + }, + ] + + with tempfile.TemporaryDirectory() as tmp: + mgr = ActionRecordingManager( + Path(tmp), surface="windows", dump_fn=None, + defer_action_evidence=True, + ) + mgr.start(entry_dump=WindowsDump()) + action = mgr.handle_raw_event( + RawUserEvent( + kind="click", surface="windows", x=181, y=45, + target={ + "control_type": "WindowControl", "name": "WeCom", + "class_name": "WeWorkWindow", + "bounds": {"left": 0, "top": 0, "right": 1440, "bottom": 900, "width": 1440, "height": 900}, + }, + meta={"uia_coordinates": {"x": 181, "y": 45}}, + ) + ) + assert action is not None + mgr.attach_eventual_evidence( + [action.id], dump=WindowsDump(), requested_monotonic=__import__("time").monotonic() + ) + self.assertEqual(action.target_snapshot.get("name"), "searchedit") + self.assertEqual(action.target_snapshot.get("control_type"), "TreeControl") + self.assertEqual(action.evidence.get("target_promotion"), "eventual_tree_coordinate_hit") + self.assertTrue(action.target_element_id) + + def test_eventual_popup_tree_promotes_generic_container_to_unique_result_row(self) -> None: + class PopupDump(Dump): + def __init__(self): + super().__init__(value="after") + self.surface = "windows" + self.driver = "uiautomation" + self.window = { + "hwnd": 30, "pid": 20, + "title": "Search", "class_name": "SearchResultWindow2", + } + self.locatable = [ + { + "control_type": "TreeControl", "name": "search_result", + "rect": {"left": 0, "top": 0, "right": 1000, "bottom": 600, "width": 1000, "height": 600}, + }, + { + "control_type": "TreeControl", "name": "user_item", "value": "Alice", + "rect": {"left": 0, "top": 50, "right": 1000, "bottom": 110, "width": 1000, "height": 60}, + }, + ] + + with tempfile.TemporaryDirectory() as tmp: + mgr = ActionRecordingManager( + Path(tmp), surface="windows", dump_fn=None, + defer_action_evidence=True, + ) + mgr.start(entry_dump=PopupDump()) + action = mgr.handle_raw_event( + RawUserEvent( + kind="click", surface="windows", x=400, y=400, + target={ + "control_type": "TreeControl", "name": "search_result", + "bounds": {"left": 0, "top": 0, "right": 1000, "bottom": 600, "width": 1000, "height": 600}, + "window_scope": {"hwnd": 30, "pid": 20, "class_name": "SearchResultWindow2"}, + }, + meta={"uia_coordinates": {"x": 400, "y": 400}}, + ) + ) + assert action is not None + mgr.attach_eventual_evidence( + [action.id], dump=PopupDump(), + requested_monotonic=__import__("time").monotonic(), + ) + self.assertEqual(action.target_snapshot.get("name"), "user_item") + self.assertEqual(action.target_snapshot.get("value"), "Alice") + self.assertEqual(action.evidence.get("target_promotion"), "eventual_tree_unique_row") + + def test_main_window_never_uses_popup_unique_row_fallback(self) -> None: + class MainDump(Dump): + def __init__(self): + super().__init__(value="after") + self.surface = "windows" + self.driver = "uiautomation" + self.window = { + "hwnd": 10, "pid": 20, "title": "WeCom", + "class_name": "WeWorkWindow", + } + self.locatable = [ + { + "control_type": "ListItemControl", "name": "listitem_content", + "rect": {"left": 0, "top": 50, "right": 300, "bottom": 110, "width": 300, "height": 60}, + } + ] + + broad = { + "control_type": "WindowControl", "name": "WeCom", + "bounds": {"left": 0, "top": 0, "right": 1400, "bottom": 900, "width": 1400, "height": 900}, + "window_scope": {"hwnd": 10, "pid": 20, "class_name": "WeWorkWindow"}, + } + with tempfile.TemporaryDirectory() as tmp: + mgr = ActionRecordingManager( + Path(tmp), surface="windows", dump_fn=None, + defer_action_evidence=True, + ) + mgr.start(entry_dump=MainDump()) + action = mgr.handle_raw_event( + RawUserEvent( + kind="click", surface="windows", x=800, y=800, + target=broad, meta={"uia_coordinates": {"x": 800, "y": 800}}, + ) + ) + assert action is not None + mgr.attach_eventual_evidence( + [action.id], dump=MainDump(), + requested_monotonic=__import__("time").monotonic(), + ) + self.assertEqual(action.target_snapshot.get("name"), "WeCom") + self.assertNotEqual( + action.evidence.get("target_promotion"), "eventual_tree_unique_row" + ) + + def test_late_enter_evidence_uses_prior_editor_action_as_context(self) -> None: + class WindowsDump(Dump): + def __init__(self): + super().__init__(value="after") + self.surface = "windows" + self.driver = "uiautomation" + self.window = {"hwnd": 10, "pid": 20, "title": "WeCom", "class_name": "WeWorkWindow"} + self.locatable = [ + { + "control_type": "TreeControl", "name": "send_rich_edit", + "rect": {"left": 100, "top": 500, "right": 900, "bottom": 700, "width": 800, "height": 200}, + } + ] + + editor = { + "control_type": "TreeControl", "name": "send_rich_edit", + "bounds": {"left": 100, "top": 500, "right": 900, "bottom": 700, "width": 800, "height": 200}, + "window_scope": {"hwnd": 10, "pid": 20, "class_name": "WeWorkWindow"}, + } + root = { + "control_type": "WindowControl", "name": "WeCom", + "bounds": {"width": 1440, "height": 900}, + "window_scope": {"hwnd": 10, "pid": 20, "class_name": "WeWorkWindow"}, + } + with tempfile.TemporaryDirectory() as tmp: + mgr = ActionRecordingManager( + Path(tmp), surface="windows", dump_fn=None, + defer_action_evidence=True, + ) + mgr.start(entry_dump=WindowsDump()) + mgr.handle_raw_event(RawUserEvent(kind="click", surface="windows", target=editor)) + enter = mgr.handle_raw_event(RawUserEvent(kind="key", surface="windows", key="Enter", target=root)) + assert enter is not None + mgr.attach_eventual_evidence( + [enter.id], dump=WindowsDump(), + requested_monotonic=__import__("time").monotonic(), + ) + self.assertEqual(enter.target_snapshot.get("name"), "send_rich_edit") + + def test_late_dynamic_message_preview_cannot_rewrite_old_root_click(self) -> None: + class WindowsDump(Dump): + def __init__(self): + super().__init__(value="after") + self.surface = "windows" + self.driver = "uiautomation" + self.window = {"hwnd": 10, "pid": 20, "title": "WeCom", "class_name": "WeWorkWindow"} + self.locatable = [ + { + "control_type": "TreeItemControl", + "name": "groupbuddydescription", + "value": "a much later message preview", + "rect": {"left": 0, "top": 0, "right": 500, "bottom": 60, "width": 500, "height": 60}, + } + ] + + root = { + "control_type": "WindowControl", "name": "WeCom", + "bounds": {"width": 1440, "height": 900}, + "window_scope": {"hwnd": 10, "pid": 20, "class_name": "WeWorkWindow"}, + } + with tempfile.TemporaryDirectory() as tmp: + mgr = ActionRecordingManager( + Path(tmp), surface="windows", dump_fn=None, + defer_action_evidence=True, + ) + mgr.start(entry_dump=WindowsDump()) + action = mgr.handle_raw_event( + RawUserEvent( + kind="click", surface="windows", x=20, y=20, + target=root, meta={"uia_coordinates": {"x": 20, "y": 20}}, + ) + ) + assert action is not None + mgr.attach_eventual_evidence( + [action.id], dump=WindowsDump(), + requested_monotonic=__import__("time").monotonic() - 10.0, + ) + self.assertEqual(action.target_snapshot.get("control_type"), "WindowControl") + self.assertEqual(action.evidence.get("target_promotion_rejected"), "late_dynamic_candidate") + + def test_windows_deferred_evidence_keeps_action_hot_path_fast(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + mgr = ActionRecordingManager( + Path(tmp), surface="windows", dump_fn=None, + defer_action_evidence=True, + ) + mgr.start(entry_dump=Dump(value="entry")) + + def slow_dump(**_kwargs): + import time + time.sleep(0.2) + return Dump(value="eventual") + + mgr.dump_fn = slow_dump + started = __import__("time").monotonic() + action = mgr.handle_raw_event( + RawUserEvent( + kind="click", surface="windows", + target={"control_type": "ButtonControl", "name": "Send"}, + ) + ) + self.assertLess(__import__("time").monotonic() - started, 0.1) + self.assertIsNotNone(action) + + def test_windows_session_buffers_event_emitted_during_startup(self) -> None: + class FakeWindowsRecorder: + def __init__(self, *, on_event, exclude_hwnds): + self.on_event = on_event + self.status = RecorderState.IDLE + + def start(self): + self.status = RecorderState.RECORDING + self.on_event( + RawUserEvent( + kind="click", surface="windows", + target={"control_type": "ButtonControl", "name": "Early"}, + ) + ) + + def stop(self): + self.status = RecorderState.STOPPED + + with tempfile.TemporaryDirectory() as tmp, patch( + "surfaces.windows.WindowsRecorder", FakeWindowsRecorder + ): + session = FlowSession( + Path(tmp), flow_name="startup", surface="windows", dump_fn=None + ) + session.start() + session._event_queue.join() + self.assertEqual(len(session.manager.action_flow.actions), 1) + session.stop() + + def test_deferred_evidence_does_not_block_action_hot_path(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + mgr = ActionRecordingManager( + root, surface="browser", dump_fn=None, defer_action_evidence=True + ) + + def slow_dump(**_kwargs): + import time + time.sleep(0.2) + return Dump(value="eventual") + + evidence = EvidenceCoordinator(mgr, slow_dump, coalesce_sec=0.03) + mgr.on_action = evidence.schedule + mgr.start(entry_dump=Dump(value="entry")) + evidence.start() + import time + started = time.monotonic() + action = mgr.handle_raw_event( + RawUserEvent( + kind="click", surface="browser", + target={"role": "button", "name": "Go", "css_count": 1}, + meta={"page_id": "page_1"}, + ) + ) + elapsed = time.monotonic() - started + self.assertLess(elapsed, 0.1) + self.assertIsNotNone(action) + assert action is not None + self.assertIsNone(action.after_state_id) + evidence.stop(timeout=3.0) + self.assertIn("eventual_observation_id", action.evidence) + + def test_weak_comment_icon_gets_submit_semantic_hint(self) -> None: + class Before: + locatable = [ + {"role": "button", "name": "提交"}, + {"role": "button", "name": "关注"}, + ] + + target = ActionRecordingManager._enrich_weak_browser_target( + { + "role": "path", + "tag": "path", + "css": "#comment-input-container .commentInput-right-ct svg path", + }, + Before(), + ) + self.assertEqual(target.get("semantic_role"), "button") + self.assertEqual(target.get("semantic_name"), "提交") + + def test_press_key_keeps_raw_event_and_target_element(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + mgr = ActionRecordingManager( + Path(tmp), surface="browser", dump_fn=None, + defer_action_evidence=True, + ) + mgr.start() + action = mgr.handle_raw_event( + RawUserEvent( + kind="key", + surface="browser", + key="Enter", + url="https://example.test/search", + frame={"url": "https://example.test/search", "name": ""}, + target={ + "role": "textbox", + "tag": "input", + "placeholder": "Search", + "stable_css": 'input[placeholder="Search"]', + "stable_css_count": 1, + "fingerprint": "input|search", + }, + meta={ + "event_id": "doc:17", + "document_id": "doc", + "page_id": "page_1", + "sequence": 17, + }, + ) + ) + self.assertIsNotNone(action) + assert action is not None + self.assertTrue(action.target_element_id) + self.assertEqual(action.raw_event["meta"]["event_id"], "doc:17") + self.assertEqual(action.raw_event["meta"]["page_id"], "page_1") + self.assertEqual(action.evidence["locator_match_count"], 1) + self.assertTrue(action.evidence["locator_unique"]) + + def test_action_capture_uses_one_dump_not_stability_poll(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + calls = [] + + def dump_fn(): + calls.append(1) + return Dump(value="after") + + mgr = ActionRecordingManager( + Path(tmp), surface="windows", dump_fn=dump_fn + ) + mgr.start(entry_dump=Dump(value="before")) + mgr.handle_raw_event( + RawUserEvent( + kind="click", surface="windows", + target={"control_type": "ButtonControl", "name": "Go"}, + ) + ) + self.assertEqual(len(calls), 1) + + def test_click_before_after_and_no_heartbeat_dup(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + dumps = [Dump(value="1"), Dump(value="1"), Dump(value="2")] + + def dump_fn(): + return dumps[min(len(dumps) - 1, dump_fn.i)] # type: ignore + + dump_fn.i = 0 # type: ignore + + mgr = ActionRecordingManager( + root, + flow_name="t", + surface="browser", + dump_fn=lambda: Dump(value="1"), + stability_timeout_sec=0.4, + ) + mgr.start(entry_dump=Dump(value="1")) + entry_states = list(mgr.action_flow.states) + self.assertEqual(len(entry_states), 1) + + # Heartbeat must not create new states/actions + mgr.heartbeat() + mgr.heartbeat() + self.assertEqual(len(mgr.action_flow.states), 1) + self.assertEqual(len(mgr.action_flow.actions), 0) + + # Click with stable dump + mgr.dump_fn = lambda: Dump(value="2", url="https://example.com/2") + act = mgr.handle_raw_event( + RawUserEvent( + kind="click", + surface="browser", + target={ + "role": "button", + "name": "Go", + "automation_id": "go", + "control_type": "button", + }, + ) + ) + self.assertIsNotNone(act) + assert act is not None + self.assertEqual(act.action_type, "click") + self.assertTrue(act.before_state_id) + self.assertTrue(act.after_state_id) + self.assertTrue((root / "action_flow.json").exists()) + self.assertTrue((root / "element_catalog.json").exists()) + + def test_click_dump_uses_event_origin_page(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + seen = [] + + def dump_fn(**kwargs): + seen.append(dict(kwargs)) + return Dump(url="https://example.com/detail") + + mgr = ActionRecordingManager( + Path(tmp), surface="browser", dump_fn=dump_fn, + stability_timeout_sec=0.2, + ) + mgr.start(entry_dump=Dump(url="https://example.com/list")) + act = mgr.handle_raw_event( + RawUserEvent( + kind="click", + surface="browser", + target={"role": "button", "name": "Open"}, + url="https://example.com/detail", + meta={ + "page_id": "page_detail", + "document_id": "doc_detail", + "frame_id": "frame_main", + }, + ) + ) + self.assertTrue(seen) + self.assertTrue(all(x.get("page_id") == "page_detail" for x in seen)) + self.assertIsNotNone(act) + assert act is not None + self.assertTrue(act.after_observation_id) + + def test_duplicate_state_still_persists_each_observation(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + service = SnapshotService(root) + first = service.capture_from_dump(Dump(value="same"), change_reason="entry") + second = service.capture_from_dump( + Dump(value="same"), change_reason="after_click", force_persist=True + ) + self.assertEqual(first.id, second.id) + self.assertEqual(len(service.state_ids()), 1) + self.assertEqual(len(service.observation_ids()), 2) + metas = list((root / "observations").glob("*/meta.json")) + self.assertEqual(len(metas), 2) + + def test_inspector_events_filtered(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + mgr = ActionRecordingManager(Path(tmp), surface="windows", dump_fn=None) + mgr.status = RecorderState.RECORDING + act = mgr.handle_raw_event( + RawUserEvent(kind="click", surface="windows", from_inspector=True) + ) + self.assertIsNone(act) + + def test_flow_session_processes_events_in_fifo_order(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + session = FlowSession(Path(tmp), flow_name="fifo", surface="browser") + session.manager.start() + session.manager._debounce_sec = 0.0 + session._start_event_consumer() + for name in ("first", "second", "third"): + self.assertTrue( + session._submit( + "event", + RawUserEvent( + kind="click", + surface="browser", + target={"role": "button", "name": name}, + ), + ) + ) + session._event_queue.join() + names = [] + for path in sorted((Path(tmp) / "actions").glob("*.json")): + import json + + data = json.loads(path.read_text(encoding="utf-8")) + names.append((data["sequence"], data["target_snapshot"]["name"])) + self.assertEqual( + [name for _, name in sorted(names)], ["first", "second", "third"] + ) + session.stop() + + def test_sensitive_value_absent_from_all_action_artifacts(self) -> None: + secret = "super-secret-password-987654321" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + mgr = ActionRecordingManager(root, surface="browser", dump_fn=None) + mgr.start() + mgr.handle_raw_event( + RawUserEvent( + kind="click", + surface="browser", + target={ + "role": "textbox", + "name": "password", + "input_type": "password", + "value": secret, + }, + meta={"raw_value": secret}, + ) + ) + persisted = "\n".join( + p.read_text(encoding="utf-8", errors="ignore") + for p in root.rglob("*") + if p.is_file() + ) + self.assertNotIn(secret, persisted) + + +class TestBrowserRecorderBridge(unittest.TestCase): + def test_pointerdown_is_fallback_when_click_never_arrives(self) -> None: + seen = [] + recorder = BrowserRecorder(on_event=seen.append) + recorder.status = RecorderState.RECORDING + recorder._pointer_fallback_sec = 0.0 + recorder._emit_dom_event( + { + "event_id": "doc:down", + "kind": "pointerdown", + "page_id": "page_1", + "document_id": "doc", + "pointer_id": 1, + "button": 0, + "x": 10, + "y": 10, + "target": {"role": "link", "tag": "a"}, + "candidate_chain": [{"role": "link", "tag": "a"}], + } + ) + recorder._flush_pending_pointerdowns() + self.assertEqual(len(seen), 1) + self.assertTrue(seen[0].meta.get("pointerdown_fallback")) + + def test_click_cancels_pointerdown_fallback(self) -> None: + seen = [] + recorder = BrowserRecorder(on_event=seen.append) + recorder.status = RecorderState.RECORDING + base = { + "page_id": "page_1", "document_id": "doc", "pointer_id": 1, + "button": 0, "x": 10, "y": 10, + "target": {"role": "button", "tag": "button"}, + "candidate_chain": [{"role": "button", "tag": "button"}], + } + recorder._emit_dom_event({**base, "event_id": "doc:down", "kind": "pointerdown"}) + recorder._emit_dom_event({**base, "event_id": "doc:click", "kind": "click"}) + recorder._flush_pending_pointerdowns(force=True) + self.assertEqual([event.kind for event in seen], ["click"]) + + def test_new_page_without_opener_still_emits_navigation(self) -> None: + seen = [] + recorder = BrowserRecorder(on_event=seen.append) + recorder.status = RecorderState.RECORDING + recorder._known_page_ids.add("page_old") + payload = { + "page_id": "page_new", "opener_page_id": "", + "url": "https://example.com/detail", "events": [], + } + with patch.object(kit_browser, "call_on_each_open_page", return_value=[payload]): + recorder._tick() + recorder._flush_pending_navigations(force=True) + self.assertEqual([event.kind for event in seen], ["navigation"]) + self.assertTrue(seen[0].meta.get("new_page")) + + def test_stop_drains_bridge_events_already_received(self) -> None: + seen = [] + recorder = BrowserRecorder(on_event=seen.append) + recorder.status = RecorderState.RECORDING + recorder._receive_bound_event( + { + "event_id": "doc:last", + "kind": "click", + "target": {"role": "button", "name": "Last"}, + } + ) + recorder.stop() + self.assertEqual([event.target.get("name") for event in seen], ["Last"]) + + def test_polled_click_is_emitted_before_due_navigation(self) -> None: + seen = [] + recorder = BrowserRecorder(on_event=seen.append) + recorder.status = RecorderState.RECORDING + recorder._pending_navigations["page_1"] = { + "due": 0.0, + "from": "https://example.com/list", + "to": "https://example.com/detail", + "page_id": "page_1", + } + payload = { + "page_id": "page_1", + "url": "https://example.com/detail", + "events": [ + { + "event_id": "doc:open", + "kind": "click", + "target": {"role": "link", "name": "Open"}, + } + ], + } + with patch.object(kit_browser, "call_on_each_open_page", return_value=[payload]): + recorder._tick() + self.assertEqual([event.kind for event in seen], ["click", "navigation"]) + + def test_bound_and_poll_fallback_event_is_deduplicated(self) -> None: + seen = [] + recorder = BrowserRecorder(on_event=seen.append) + recorder.status = RecorderState.RECORDING + event = { + "event_id": "doc:1", + "document_id": "doc", + "sequence": 1, + "kind": "click", + "target": {"role": "button", "name": "Go"}, + } + recorder._emit_dom_event(dict(event)) + recorder._emit_dom_event(dict(event)) + self.assertEqual(len(seen), 1) + + def test_bridge_preserves_page_and_frame_identity(self) -> None: + class Page: + def is_closed(self): + return False + + page = Page() + frame = object() + seen = [] + with kit_browser._bridge_lock: + kit_browser._bridge_sink = seen.append + try: + kit_browser._dispatch_bridge_event( + {"page": page, "frame": frame}, {"kind": "click"} + ) + finally: + kit_browser.clear_event_bridge() + kit_browser._page_ids.clear() + kit_browser._page_refs.clear() + kit_browser._pages_by_id.clear() + kit_browser._frame_ids.clear() + kit_browser._frame_refs.clear() + self.assertEqual(len(seen), 1) + self.assertTrue(seen[0]["page_id"].startswith("page_")) + self.assertIn(":frame_", seen[0]["frame_id"]) + + def test_svg_hit_keeps_promoted_action_target(self) -> None: + seen = [] + recorder = BrowserRecorder(on_event=seen.append) + recorder.status = RecorderState.RECORDING + recorder._emit_dom_event( + { + "event_id": "doc:svg", + "kind": "click", + "page_id": "page_video", + "target": {"role": "button", "name": "Play", "tag": "button"}, + "hit_target": {"role": "path", "tag": "path"}, + "action_target": {"role": "button", "name": "Play", "tag": "button"}, + } + ) + self.assertEqual(seen[0].target.get("role"), "button") + self.assertEqual(seen[0].meta.get("hit_target", {}).get("tag"), "path") + self.assertEqual(seen[0].meta.get("page_id"), "page_video") + + def test_navigation_failure_is_not_swallowed(self) -> None: + class Page: + url = "about:blank" + + def goto(self, *args, **kwargs): + raise RuntimeError("network down") + + with self.assertRaisesRegex(RuntimeError, "ERROR:NAVIGATION_FAILED"): + kit_browser._goto_safe(Page(), "https://example.invalid") + + +class TestWeComAdapter(unittest.TestCase): + def test_profiles(self) -> None: + self.assertEqual( + wecom_adapter.wake_profile(class_name="WeWorkWindow"), "wecom" + ) + self.assertEqual( + wecom_adapter.wake_profile(class_name="SearchResultWindow2"), + "wecom_search", + ) + self.assertEqual(resolve_wake_profile(class_name="WeWorkWindow"), "wecom") + + def test_cache_keys_isolated(self) -> None: + k1 = wecom_adapter.cache_key(profile="wecom", hwnd=100) + k2 = wecom_adapter.cache_key(profile="wecom_search", hwnd=200) + k3 = _cache_key("wecom", 100) + self.assertNotEqual(k1, k2) + self.assertEqual(k1, k3) + # Main warm key must not equal search key even with hwnd=0 confusion + self.assertNotEqual( + wecom_adapter.cache_key(profile="wecom", hwnd=1), + wecom_adapter.cache_key(profile="wecom_search", hwnd=1), + ) + + def test_proven_warm_provider_is_pinned_for_recorder_flow(self) -> None: + _caches.clear() + self.assertTrue( + note_provider_warm( + class_name="WeWorkWindow", title="WeCom", hwnd=100, findall=120 + ) + ) + report = auto_ensure_if_wecom( + class_name="WeWorkWindow", title="WeCom", hwnd=100, + findall_hint=0, + ) + self.assertTrue(report.get("ok")) + self.assertEqual(report.get("reason"), "cache_warm") + self.assertFalse(report.get("wake_applied")) + + +class TestV1FlowStillCreated(unittest.TestCase): + def test_create_flow_has_steps_and_v2(self) -> None: + import artifacts as art + + with tempfile.TemporaryDirectory() as tmp: + flow_dir = art.create_flow(Path(tmp), flow_name="demo", recording_mode="auto") + self.assertTrue((flow_dir / "steps").is_dir()) + self.assertTrue((flow_dir / "states").is_dir()) + self.assertTrue((flow_dir / "observations").is_dir()) + self.assertTrue((flow_dir / "journal").is_dir()) + self.assertTrue((flow_dir / "diagnostics").is_dir()) + flow = art.load_flow(flow_dir) + self.assertEqual(flow["schema"], "openclaw.ui_inspector.flow.v1") + self.assertIn("v2", flow) + self.assertEqual(flow["v2"]["action_flow"], "action_flow.json") + self.assertEqual(flow["v2"]["raw_event_journal"], "journal/raw_events.jsonl") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/ui_inspector/tests/unit/__init__.py b/tools/ui_inspector/tests/unit/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tools/ui_inspector/tests/unit/__init__.py @@ -0,0 +1 @@ + diff --git a/tools/ui_inspector/tests/unit/core/__init__.py b/tools/ui_inspector/tests/unit/core/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tools/ui_inspector/tests/unit/core/__init__.py @@ -0,0 +1 @@ + diff --git a/tools/ui_inspector/tests/unit/core/test_event_journal_identity.py b/tools/ui_inspector/tests/unit/core/test_event_journal_identity.py new file mode 100644 index 0000000..80262f0 --- /dev/null +++ b/tools/ui_inspector/tests/unit/core/test_event_journal_identity.py @@ -0,0 +1,30 @@ +import json +from pathlib import Path + +from core.pipeline.event_journal import EventJournal +from core.policies.element_identity import element_identity_key + + +def test_journal_is_append_only(tmp_path: Path) -> None: + journal = EventJournal(tmp_path) + assert journal.append({"kind": "pointerdown"}) == 1 + assert journal.append({"kind": "click"}) == 2 + rows = [json.loads(line) for line in journal.path.read_text(encoding="utf-8").splitlines()] + assert [row["journal_sequence"] for row in rows] == [1, 2] + assert [row["event"]["kind"] for row in rows] == ["pointerdown", "click"] + + +def test_focus_classes_and_query_strings_do_not_split_identity() -> None: + first = { + "role": "textbox", "tag": "input", "placeholder": "Search", + "css": "#header input.search", + } + focused = { + **first, + "class_name": "search focused hash-123", + "css": "#header input.search.focused.hash-123", + } + one = element_identity_key(first, surface="browser", page_url="https://example.com/home?a=1") + two = element_identity_key(focused, surface="browser", page_url="https://example.com/results?q=x") + assert one == two + diff --git a/tools/ui_inspector/tests/unit/core/test_target_scoring.py b/tools/ui_inspector/tests/unit/core/test_target_scoring.py new file mode 100644 index 0000000..666dc7f --- /dev/null +++ b/tools/ui_inspector/tests/unit/core/test_target_scoring.py @@ -0,0 +1,70 @@ +from core.policies.target_scoring import resolve_browser_target + + +def test_huge_control_named_ancestor_never_beats_comment_hit() -> None: + hit = { + "tag": "path", "role": "path", "cursor": "pointer", + "rect": {"left": 1047, "top": 545, "right": 1070, "bottom": 568, "width": 23, "height": 23}, + "css": "#comment-input-container .send > svg > path", + } + huge = { + "tag": "div", "role": "div", "class_name": "playerControlHeight", + "rect": {"left": 160, "top": -244, "right": 1440, "bottom": 6698, "width": 1280, "height": 6942}, + "css": "#root .playerControlHeight", + } + target, scored = resolve_browser_target(hit, [hit, huge], x=1056, y=558) + assert target["tag"] == "path" + assert any("oversized_ancestor" in row["reasons"] for row in scored) + + +def test_nearest_semantic_wrapper_beats_svg_path() -> None: + hit = { + "tag": "path", "role": "path", "cursor": "pointer", + "rect": {"left": 10, "top": 10, "right": 30, "bottom": 30, "width": 20, "height": 20}, + } + wrapper = { + "tag": "span", "role": "span", "cursor": "pointer", "class_name": "sendControl", + "rect": {"left": 5, "top": 5, "right": 35, "bottom": 35, "width": 30, "height": 30}, + } + target, _ = resolve_browser_target(hit, [hit, wrapper], x=20, y=20) + assert target["class_name"] == "sendControl" + + +def test_contenteditable_root_beats_transient_inner_block() -> None: + inner = { + "tag": "div", "role": "textbox", "contenteditable": True, + "contenteditable_attr": False, + "rect": {"left": 10, "top": 10, "right": 300, "bottom": 35, "width": 290, "height": 25}, + } + root = { + "tag": "div", "role": "combobox", "explicit_role": "combobox", + "contenteditable": True, "contenteditable_attr": True, + "rect": {"left": 10, "top": 10, "right": 300, "bottom": 35, "width": 290, "height": 25}, + } + target, _ = resolve_browser_target(inner, [inner, root], x=20, y=20) + assert target["contenteditable_attr"] is True + + +def test_unique_clickable_wrapper_beats_ambiguous_graphic_leaf() -> None: + rect_path = {"left": 1047, "top": 345, "right": 1070, "bottom": 368, "width": 23, "height": 23} + rect_wrapper = {"left": 1041, "top": 338, "right": 1077, "bottom": 374, "width": 36, "height": 36} + path = { + "tag": "path", "role": "path", "cursor": "pointer", + "rect": rect_path, "css_count": 2, + } + svg = { + "tag": "svg", "role": "svg", "cursor": "pointer", + "rect": rect_wrapper, "css_count": 1, + } + wrapper = { + "tag": "span", "role": "span", "cursor": "pointer", + "rect": rect_wrapper, "css_count": 1, + "stable_css": "#comment-form > div.controls > span", + "stable_css_count": 1, + } + target, scored = resolve_browser_target( + path, [path, svg, wrapper], x=1055, y=352 + ) + assert target["tag"] == "span" + assert any("graphic_leaf" in row["reasons"] for row in scored) + assert any("event_time_unique_locator" in row["reasons"] for row in scored) diff --git a/tools/ui_inspector/tests/unit/test_app_layout.py b/tools/ui_inspector/tests/unit/test_app_layout.py new file mode 100644 index 0000000..4feaf18 --- /dev/null +++ b/tools/ui_inspector/tests/unit/test_app_layout.py @@ -0,0 +1,120 @@ +import json +import sys +from pathlib import Path + +_HERE = Path(__file__).resolve().parents[2] +if str(_HERE) not in sys.path: + sys.path.insert(0, str(_HERE)) + +from app import ( + _DEFAULT_FLOW_NAME_SLUGS, + _default_window_geometry, + _flow_name_for_mode, + _format_geometry, + _format_live_action, + _merge_current_live_action, +) +import artifacts as art +import target +import user_prefs + + +def test_default_geometry_is_narrow_tall_and_leaves_title_bar_clear() -> None: + width, height, x, y = _default_window_geometry((0, 0, 1920, 1040)) + + assert (width, height) == (576, 951) + assert (x, y) == (1328, 73) + assert y >= 64 + assert x + width == 1920 - 16 + + +def test_default_geometry_supports_small_and_left_side_monitors() -> None: + width, height, x, y = _default_window_geometry( + (-1366, 0, 0, 728), side="left" + ) + + assert (width, height) == (560, 648) + assert (x, y) == (-1350, 64) + assert _format_geometry(width, height, x, y) == "560x648-1350+64" + + +def test_flow_name_defaults_follow_pick_mode() -> None: + assert _flow_name_for_mode(target.SURFACE_BROWSER) == "browser" + assert _flow_name_for_mode(target.SURFACE_WINDOWS) == "desktop" + assert _flow_name_for_mode(target.SURFACE_ANDROID) == "mobile" + assert _DEFAULT_FLOW_NAME_SLUGS == frozenset({"browser", "desktop", "mobile"}) + + +def test_local_ts_is_compact_yyyymmddhhmmss() -> None: + stamp = art._local_ts() + assert len(stamp) == 14 + assert stamp.isdigit() + assert "T" not in stamp + assert "+" not in stamp + + +def test_create_flow_dir_uses_compact_timestamp(tmp_path: Path) -> None: + flow_dir = art.create_flow(tmp_path, flow_name="browser") + name = flow_dir.name + assert name.endswith("_browser") + stamp = name[: -len("_browser")] + assert len(stamp) == 14 + assert stamp.isdigit() + + +def test_hidden_legacy_recording_preference_migrates_to_auto( + tmp_path: Path, monkeypatch +) -> None: + prefs_path = tmp_path / "user_prefs.json" + prefs_path.write_text( + json.dumps( + { + "mode": "browser", + "start_url": "https://example.com", + "recording_mode": "legacy_interval", + } + ), + encoding="utf-8", + ) + monkeypatch.setattr(user_prefs, "_PREFS_PATH", prefs_path) + + assert user_prefs.load_prefs()["recording_mode"] == "auto" + + +def test_live_action_formatter_is_surface_neutral_and_two_phase() -> None: + text = _format_live_action( + { + "sequence": 3, + "action_type": "click", + "phase": "captured", + "surface": "browser", + "target_name": "提交", + "target_type": "button", + } + ) + assert "#3 点击" in text + assert "browser" in text + assert "证据采集中" in text + assert "等待异步证据" in text + + +def test_current_live_action_overwrites_without_allowing_late_evidence_to_regress() -> None: + first = {"id": "action-1", "sequence": 1, "phase": "captured"} + first_evidence = {"id": "action-1", "sequence": 1, "phase": "evidence"} + second = {"id": "action-2", "sequence": 2, "phase": "captured"} + + current = _merge_current_live_action(None, first) + current = _merge_current_live_action(current, first_evidence) + assert current == first_evidence + + current = _merge_current_live_action(current, second) + assert current == second + + current = _merge_current_live_action(current, first_evidence) + assert current == second + + +def test_current_live_action_ignores_updates_without_an_action_id() -> None: + current = {"id": "action-2", "sequence": 2, "phase": "captured"} + + assert _merge_current_live_action(current, {"sequence": 3}) is current diff --git a/tools/ui_inspector/ui/__init__.py b/tools/ui_inspector/ui/__init__.py new file mode 100644 index 0000000..8acdbd3 --- /dev/null +++ b/tools/ui_inspector/ui/__init__.py @@ -0,0 +1,2 @@ +"""User-interface package; legacy app.py remains the compatibility entry.""" + diff --git a/tools/ui_inspector/ui/controller.py b/tools/ui_inspector/ui/controller.py new file mode 100644 index 0000000..72115a3 --- /dev/null +++ b/tools/ui_inspector/ui/controller.py @@ -0,0 +1,10 @@ +"""Application-facing session facade. + +The Tk view remains in app.py during the compatibility migration; new callers +should depend on this facade instead of constructing recorder internals. +""" + +from flow_session import FlowSession + +__all__ = ["FlowSession"] + diff --git a/tools/ui_inspector/user_prefs.py b/tools/ui_inspector/user_prefs.py new file mode 100644 index 0000000..1c19830 --- /dev/null +++ b/tools/ui_inspector/user_prefs.py @@ -0,0 +1,88 @@ +"""Persist UI Inspector prefs (pick mode + start URL + recording mode) across runs.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import target + +_HERE = Path(__file__).resolve().parent +_PREFS_PATH = _HERE / "user_prefs.json" + +DEFAULT_MODE = target.SURFACE_BROWSER +# 起始 URL：默认空；仅记住用户主动填写过的值 +DEFAULT_START_URL = "" +DEFAULT_RECORDING_MODE = "auto" + +_VALID_MODES = { + target.SURFACE_BROWSER, + target.SURFACE_WINDOWS, + target.SURFACE_ANDROID, # UI label「手机」 +} + +_VALID_RECORDING = {"auto", "guided", "checkpoint", "legacy_interval"} + + +def prefs_path() -> Path: + return _PREFS_PATH + + +def load_prefs() -> dict[str, Any]: + raw: dict[str, Any] = {} + try: + if _PREFS_PATH.is_file(): + data = json.loads(_PREFS_PATH.read_text(encoding="utf-8")) + if isinstance(data, dict): + raw = data + except Exception: + raw = {} + mode = str(raw.get("mode") or DEFAULT_MODE).strip().lower() + if mode in {"mobile", "phone", "ios"}: + mode = target.SURFACE_ANDROID + if mode not in _VALID_MODES: + mode = DEFAULT_MODE + # 允许空字符串；未写过 start_url 键时也视为空 + if "start_url" in raw: + start_url = str(raw.get("start_url") or "").strip() + else: + start_url = DEFAULT_START_URL + # v0.14.3 standardizes the public UI on automatic action recording. + # Older values remain accepted by lower-level capture APIs, but must not + # silently put the simplified UI into a hidden legacy mode. + recording_mode = DEFAULT_RECORDING_MODE + return { + "mode": mode, + "start_url": start_url, + "recording_mode": recording_mode, + } + + +def save_prefs( + *, + mode: str, + start_url: str, + recording_mode: str = DEFAULT_RECORDING_MODE, +) -> None: + mode_n = (mode or DEFAULT_MODE).strip().lower() + if mode_n in {"mobile", "phone", "ios"}: + mode_n = target.SURFACE_ANDROID + if mode_n not in _VALID_MODES: + mode_n = DEFAULT_MODE + url = (start_url or "").strip() + rec = (recording_mode or DEFAULT_RECORDING_MODE).strip().lower() + if rec not in _VALID_RECORDING: + rec = DEFAULT_RECORDING_MODE + payload = { + "mode": mode_n, + "start_url": url, + "recording_mode": rec, + } + try: + _PREFS_PATH.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + except Exception: + pass