From 2c9bf560544137f9be279a351b04970c41253872 Mon Sep 17 00:00:00 2001 From: chendelian <116870791@qq.com> Date: Fri, 7 Aug 2026 11:20:23 +0800 Subject: [PATCH] chore: remove tools/ui_inspector; point docs to jiangchang-ui-recorder. Product lives at https://git.jc2009.com/client-jiangchang/jiangchang-ui-recorder; prior commit keeps an archive snapshot. Co-authored-by: Cursor --- development/RPA_ELEMENT_LOCATING.md | 2 + tools/README.md | 13 +- tools/ui_inspector/.gitignore | 7 - tools/ui_inspector/README.md | 189 -- tools/ui_inspector/__init__.py | 3 - tools/ui_inspector/__main__.py | 12 - tools/ui_inspector/adapters/__init__.py | 7 - tools/ui_inspector/adapters/base.py | 17 - tools/ui_inspector/adapters/registry.py | 31 - tools/ui_inspector/adapters/wecom.py | 103 - tools/ui_inspector/app.py | 2001 ----------------- tools/ui_inspector/artifacts.py | 735 ------ tools/ui_inspector/compatibility/__init__.py | 2 - tools/ui_inspector/compatibility/v1_reader.py | 6 - tools/ui_inspector/compatibility/v1_writer.py | 6 - tools/ui_inspector/core/__init__.py | 2 - tools/ui_inspector/core/models/__init__.py | 16 - tools/ui_inspector/core/pipeline/__init__.py | 6 - .../core/pipeline/event_journal.py | 49 - tools/ui_inspector/core/policies/__init__.py | 7 - .../core/policies/element_identity.py | 62 - .../core/policies/target_scoring.py | 145 -- tools/ui_inspector/desktop_uia.py | 1731 -------------- tools/ui_inspector/diagnostics/__init__.py | 6 - .../diagnostics/capture_validator.py | 134 -- .../diagnostics/replay_preflight.py | 133 -- tools/ui_inspector/drivers/__init__.py | 29 - tools/ui_inspector/drivers/base.py | 38 - tools/ui_inspector/drivers/browser_pw.py | 90 - tools/ui_inspector/drivers/mobile.py | 30 - tools/ui_inspector/drivers/windows_uia.py | 58 - tools/ui_inspector/evidence/__init__.py | 6 - tools/ui_inspector/evidence/coordinator.py | 255 --- tools/ui_inspector/flow_session.py | 408 ---- tools/ui_inspector/kit_browser.py | 729 ------ tools/ui_inspector/kit_wake.py | 284 --- tools/ui_inspector/locators/__init__.py | 13 - tools/ui_inspector/locators/base.py | 74 - tools/ui_inspector/locators/browser.py | 190 -- tools/ui_inspector/locators/catalog.py | 294 --- tools/ui_inspector/locators/validator.py | 50 - tools/ui_inspector/locators/windows.py | 180 -- tools/ui_inspector/models/__init__.py | 22 - tools/ui_inspector/models/action.py | 119 - tools/ui_inspector/models/element.py | 126 -- tools/ui_inspector/models/flow.py | 11 - tools/ui_inspector/models/locator.py | 40 - tools/ui_inspector/models/observation.py | 47 - tools/ui_inspector/models/serialize.py | 64 - tools/ui_inspector/models/state.py | 175 -- tools/ui_inspector/recorder/__init__.py | 13 - tools/ui_inspector/recorder/base.py | 79 - tools/ui_inspector/recorder/browser.py | 740 ------ .../ui_inspector/recorder/input_aggregator.py | 234 -- tools/ui_inspector/recorder/manager.py | 1220 ---------- tools/ui_inspector/recorder/windows.py | 1711 -------------- tools/ui_inspector/redaction.py | 206 -- tools/ui_inspector/requirements.txt | 4 - tools/ui_inspector/run.py | 10 - tools/ui_inspector/skip_policy.py | 89 - tools/ui_inspector/smoke_check.py | 383 ---- tools/ui_inspector/snapshot_service.py | 400 ---- tools/ui_inspector/state_diff.py | 275 --- tools/ui_inspector/storage/__init__.py | 6 - tools/ui_inspector/storage/capture_reader.py | 20 - tools/ui_inspector/storage/capture_writer.py | 6 - tools/ui_inspector/surfaces/__init__.py | 2 - tools/ui_inspector/surfaces/base.py | 19 - .../ui_inspector/surfaces/browser/__init__.py | 6 - .../surfaces/browser/dump_driver.py | 6 - .../surfaces/browser/locator_compiler.py | 6 - .../surfaces/browser/locator_validator.py | 6 - .../surfaces/browser/page_registry.py | 6 - .../ui_inspector/surfaces/browser/recorder.py | 6 - .../ui_inspector/surfaces/browser/runtime.py | 23 - .../ui_inspector/surfaces/mobile/__init__.py | 6 - tools/ui_inspector/surfaces/registry.py | 22 - .../ui_inspector/surfaces/windows/__init__.py | 6 - .../surfaces/windows/dump_driver.py | 4 - .../surfaces/windows/locator_compiler.py | 4 - .../ui_inspector/surfaces/windows/recorder.py | 4 - tools/ui_inspector/target.py | 82 - tools/ui_inspector/tests/__init__.py | 0 tools/ui_inspector/tests/contract/__init__.py | 1 - .../tests/contract/test_capture_contract.py | 207 -- .../tests/test_locators_catalog.py | 215 -- .../tests/test_models_fingerprint_diff.py | 295 --- .../tests/test_recorder_adapters.py | 1513 ------------- tools/ui_inspector/tests/unit/__init__.py | 1 - .../ui_inspector/tests/unit/core/__init__.py | 1 - .../unit/core/test_event_journal_identity.py | 30 - .../tests/unit/core/test_target_scoring.py | 70 - .../tests/unit/test_app_layout.py | 120 - tools/ui_inspector/ui/__init__.py | 2 - tools/ui_inspector/ui/controller.py | 10 - tools/ui_inspector/user_prefs.py | 88 - 96 files changed, 9 insertions(+), 16905 deletions(-) delete mode 100644 tools/ui_inspector/.gitignore delete mode 100644 tools/ui_inspector/README.md delete mode 100644 tools/ui_inspector/__init__.py delete mode 100644 tools/ui_inspector/__main__.py delete mode 100644 tools/ui_inspector/adapters/__init__.py delete mode 100644 tools/ui_inspector/adapters/base.py delete mode 100644 tools/ui_inspector/adapters/registry.py delete mode 100644 tools/ui_inspector/adapters/wecom.py delete mode 100644 tools/ui_inspector/app.py delete mode 100644 tools/ui_inspector/artifacts.py delete mode 100644 tools/ui_inspector/compatibility/__init__.py delete mode 100644 tools/ui_inspector/compatibility/v1_reader.py delete mode 100644 tools/ui_inspector/compatibility/v1_writer.py delete mode 100644 tools/ui_inspector/core/__init__.py delete mode 100644 tools/ui_inspector/core/models/__init__.py delete mode 100644 tools/ui_inspector/core/pipeline/__init__.py delete mode 100644 tools/ui_inspector/core/pipeline/event_journal.py delete mode 100644 tools/ui_inspector/core/policies/__init__.py delete mode 100644 tools/ui_inspector/core/policies/element_identity.py delete mode 100644 tools/ui_inspector/core/policies/target_scoring.py delete mode 100644 tools/ui_inspector/desktop_uia.py delete mode 100644 tools/ui_inspector/diagnostics/__init__.py delete mode 100644 tools/ui_inspector/diagnostics/capture_validator.py delete mode 100644 tools/ui_inspector/diagnostics/replay_preflight.py delete mode 100644 tools/ui_inspector/drivers/__init__.py delete mode 100644 tools/ui_inspector/drivers/base.py delete mode 100644 tools/ui_inspector/drivers/browser_pw.py delete mode 100644 tools/ui_inspector/drivers/mobile.py delete mode 100644 tools/ui_inspector/drivers/windows_uia.py delete mode 100644 tools/ui_inspector/evidence/__init__.py delete mode 100644 tools/ui_inspector/evidence/coordinator.py delete mode 100644 tools/ui_inspector/flow_session.py delete mode 100644 tools/ui_inspector/kit_browser.py delete mode 100644 tools/ui_inspector/kit_wake.py delete mode 100644 tools/ui_inspector/locators/__init__.py delete mode 100644 tools/ui_inspector/locators/base.py delete mode 100644 tools/ui_inspector/locators/browser.py delete mode 100644 tools/ui_inspector/locators/catalog.py delete mode 100644 tools/ui_inspector/locators/validator.py delete mode 100644 tools/ui_inspector/locators/windows.py delete mode 100644 tools/ui_inspector/models/__init__.py delete mode 100644 tools/ui_inspector/models/action.py delete mode 100644 tools/ui_inspector/models/element.py delete mode 100644 tools/ui_inspector/models/flow.py delete mode 100644 tools/ui_inspector/models/locator.py delete mode 100644 tools/ui_inspector/models/observation.py delete mode 100644 tools/ui_inspector/models/serialize.py delete mode 100644 tools/ui_inspector/models/state.py delete mode 100644 tools/ui_inspector/recorder/__init__.py delete mode 100644 tools/ui_inspector/recorder/base.py delete mode 100644 tools/ui_inspector/recorder/browser.py delete mode 100644 tools/ui_inspector/recorder/input_aggregator.py delete mode 100644 tools/ui_inspector/recorder/manager.py delete mode 100644 tools/ui_inspector/recorder/windows.py delete mode 100644 tools/ui_inspector/redaction.py delete mode 100644 tools/ui_inspector/requirements.txt delete mode 100644 tools/ui_inspector/run.py delete mode 100644 tools/ui_inspector/skip_policy.py delete mode 100644 tools/ui_inspector/smoke_check.py delete mode 100644 tools/ui_inspector/snapshot_service.py delete mode 100644 tools/ui_inspector/state_diff.py delete mode 100644 tools/ui_inspector/storage/__init__.py delete mode 100644 tools/ui_inspector/storage/capture_reader.py delete mode 100644 tools/ui_inspector/storage/capture_writer.py delete mode 100644 tools/ui_inspector/surfaces/__init__.py delete mode 100644 tools/ui_inspector/surfaces/base.py delete mode 100644 tools/ui_inspector/surfaces/browser/__init__.py delete mode 100644 tools/ui_inspector/surfaces/browser/dump_driver.py delete mode 100644 tools/ui_inspector/surfaces/browser/locator_compiler.py delete mode 100644 tools/ui_inspector/surfaces/browser/locator_validator.py delete mode 100644 tools/ui_inspector/surfaces/browser/page_registry.py delete mode 100644 tools/ui_inspector/surfaces/browser/recorder.py delete mode 100644 tools/ui_inspector/surfaces/browser/runtime.py delete mode 100644 tools/ui_inspector/surfaces/mobile/__init__.py delete mode 100644 tools/ui_inspector/surfaces/registry.py delete mode 100644 tools/ui_inspector/surfaces/windows/__init__.py delete mode 100644 tools/ui_inspector/surfaces/windows/dump_driver.py delete mode 100644 tools/ui_inspector/surfaces/windows/locator_compiler.py delete mode 100644 tools/ui_inspector/surfaces/windows/recorder.py delete mode 100644 tools/ui_inspector/target.py delete mode 100644 tools/ui_inspector/tests/__init__.py delete mode 100644 tools/ui_inspector/tests/contract/__init__.py delete mode 100644 tools/ui_inspector/tests/contract/test_capture_contract.py delete mode 100644 tools/ui_inspector/tests/test_locators_catalog.py delete mode 100644 tools/ui_inspector/tests/test_models_fingerprint_diff.py delete mode 100644 tools/ui_inspector/tests/test_recorder_adapters.py delete mode 100644 tools/ui_inspector/tests/unit/__init__.py delete mode 100644 tools/ui_inspector/tests/unit/core/__init__.py delete mode 100644 tools/ui_inspector/tests/unit/core/test_event_journal_identity.py delete mode 100644 tools/ui_inspector/tests/unit/core/test_target_scoring.py delete mode 100644 tools/ui_inspector/tests/unit/test_app_layout.py delete mode 100644 tools/ui_inspector/ui/__init__.py delete mode 100644 tools/ui_inspector/ui/controller.py delete mode 100644 tools/ui_inspector/user_prefs.py diff --git a/development/RPA_ELEMENT_LOCATING.md b/development/RPA_ELEMENT_LOCATING.md index 93a7a3b..8ecb1c3 100644 --- a/development/RPA_ELEMENT_LOCATING.md +++ b/development/RPA_ELEMENT_LOCATING.md @@ -43,6 +43,8 @@ 未完成第 2～6 步时，代码只能保留明确的 `TODO: NEED_REAL_UI_INSPECTION`，不得写“看起来可能正确”的 selector。 +开发期推荐使用独立仓 [jiangchang-ui-recorder](https://git.jc2009.com/client-jiangchang/jiangchang-ui-recorder)（本机常见路径 `D:\OpenClaw\client-jiangchang\jiangchang-ui-recorder`，`python run.py`）：以 **Raw Event Journal → Action → Element Catalog → 异步 Observation** 录制，优先把 `action_flow.json` 与 `element_catalog.json` 交给 AI 编程工具；原始事件可重新归并，tree/index 作为诊断证据。默认不再让完整页面 dump 阻塞动作采集。本模板仓内的 `tools/ui_inspector` 已移除，勿再从 skill-template 启动。 + ## 3. 元素清单 每个关键输入、按钮、列表、弹窗、成功状态和错误状态都必须登记。推荐 YAML： diff --git a/tools/README.md b/tools/README.md index 5a807a0..08cce15 100644 --- a/tools/README.md +++ b/tools/README.md @@ -1,13 +1,14 @@ # skill-template 工具 -## 桌面 UIA 拾取（开发期） +## UI 拾取（开发期） -```powershell -cd D:\OpenClaw\client-commons\skill-template -python tools/ui_inspector/run.py --scene demo -``` +`tools/ui_inspector` 已迁出本模板仓。请使用独立产品仓： -说明见 [`ui_inspector/README.md`](ui_inspector/README.md)。企微暖树依赖本机 `jiangchang-platform-kit>=1.2.5`（与技能同调 `ensure_ui_ready`）。 +- Gitea：https://git.jc2009.com/client-jiangchang/jiangchang-ui-recorder +- 本机常见路径：`D:\OpenClaw\client-jiangchang\jiangchang-ui-recorder` +- 启动：`python run.py` + +模板仓历史中仍保留删除前的归档提交（可按需从 Git 恢复旧副本）；日常开发与运行以独立仓为准。 ## 业务仓也可以直接 clone（来源 A） diff --git a/tools/ui_inspector/.gitignore b/tools/ui_inspector/.gitignore deleted file mode 100644 index 1bbae20..0000000 --- a/tools/ui_inspector/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -# 仅本工具目录使用；不修改 skill-template 根 .gitignore -_captures/ -_browser_profile/ -user_prefs.json -__pycache__/ -*.pyc -.pytest_cache/ diff --git a/tools/ui_inspector/README.md b/tools/ui_inspector/README.md deleted file mode 100644 index 207b8c1..0000000 --- a/tools/ui_inspector/README.md +++ /dev/null @@ -1,189 +0,0 @@ -# OpenClaw UI Inspector（UI Automation Recorder v0.16.0） - -默认窗口采用窄长控制台布局：停靠到鼠标所在显示器右侧，顶部预留目标应用标题栏安全区；也可在界面中一键切换到左侧。 - -> **定位**：以人工操作为主线的开发期录制器，产出可交给 AI 编程工具的 Action Flow + Element Catalog。 -> **不是**：每 2 秒无条件 dump 页面的流水线（该行为仅保留为 Legacy 模式）。 - -## 核心模型 - -```text -人工操作 - → Raw Event Journal（先落事实，不阻塞） - → Action Reducer（去重、输入事务、pointerdown/click 合并） - → Target Resolver（距离 + 面积 + 语义） - → ActionRecord + Element Catalog - → 异步合并式 Observation / State / Screenshot -``` - -| 概念 | 作用 | -|------|------| -| **Action** | 流程主线（click / fill / press_key / checkpoint …） | -| **State** | 动作前后上下文与证据（tree/index/screenshot） | -| **Element Catalog** | 开发期与运行期共用的元素资产 | -| **Locator** | 结构化属性 + 编译后的 expression；须验证匹配数 | -| **原始 dump** | 诊断证据；不再把每次 heartbeat 登记为流程步骤 | -| **Raw Event Journal** | 可重算的原始事实；Reducer 出错时无需重走人工流程 | - -## 启动 - -```powershell -cd D:\OpenClaw\client-commons\skill-template -python tools/ui_inspector/run.py --scene demo -``` - -依赖： - -```powershell -pip install -r tools/ui_inspector/requirements.txt -pip install "jiangchang-platform-kit>=1.2.11" -``` - -## 录制流程 - -主界面固定使用自动动作录制，只保留 **开始 / 暂停（继续）/ 结束**。Recorder 立即记录 Action，完整证据由独立后台队列异步合并采集。Inspector 自身窗口上的点击和输入不会写入目标 Action Flow。 - -“运行状态与拾取结果”会对浏览器和桌面统一采用单步骤覆盖式显示；新动作替换上一动作，同一动作的异步证据返回后在原位置更新定位器、唯一性或失败原因。迟到的旧动作证据不会覆盖更新的步骤，完整历史始终保存在 `action_flow.json`、`actions/` 与 `element_catalog.json` 中。 - -引导、手动检查点和 Legacy 间隔 dump 仅作为底层兼容与开发诊断能力保留，不进入普通用户流程；旧偏好会自动迁移回自动录制。 - -## 运行时稳定性 - -- Tk 主线程只处理界面；导航、Dump、截图、动作落盘均在后台执行。 -- 后台结果通过 UI 消息队列回传，后台线程不读取 Tk Variable 或 Widget。 -- 原始事件先追加到 `journal/raw_events.jsonl`，再进入语义 Action FIFO。 -- Action 热路径不执行完整页面 dump；慢速证据在独立队列按页面合并采集。 -- 异步证据标记为 `coalesced_eventual`，不会伪装成事件时刻的精确 after state。 -- Playwright 命令均有超时；导航失败不会被伪装成“已就绪”。 -- 浏览器使用 context 级事件桥并保留轮询回退，整页跳转前的 click 不随旧 document 丢失。 - -## 拾取通道 - -- **浏览器**：Inspector 自管 Playwright；起始 URL 可选；Browser Recorder（捕获阶段监听，不 preventDefault） - - 动作后的 Dump / 截图绑定事件来源 `page_id`；手动检查点才按当前可见标签页回退，不再固定 `pages[0]` - - 列表页新开详情标签后，切到详情再 dump/录制即可；关闭详情回到列表同理 - - Recorder 会在所有已打开标签上安装监听并汇总事件 - - 输入框、`contenteditable`、SVG/图标点击会沿 `composedPath` 提升到真实可操作目标；原始命中节点仍保留作诊断证据 - - 语义定位不足时生成完整 CSS 路径，并在事件来源页面实时验证唯一性后才进入 Locator Catalog - - 链接保留 `href`，可编辑区生成稳定祖先作用域选择器；事件时刻的唯一性计数随快照保存，避免跳转后误判 - - `press_key` 保留原始事件、Page/Document 身份和目标元素引用，不再依赖前一步推断焦点 - - 页面跳转采用短暂因果重排，确保触发跳转的 click 先于 navigation；新标签同时保留 opener `page_id` - - 同时监听 `pointerdown` 与 `click`：正常 click 会去重；页面提前卸载时 pointerdown 作为保底 Action - - 通用目标评分限制祖先距离与面积，不会把小图标提升成整页容器 - - 目标解析联合考虑语义、几何、事件时刻唯一性和定位器稳定性；SVG/path 仅作命中证据，优先选择唯一的可点击包装节点 - - 输入事务与 Element Catalog 共用稳定元素身份，焦点/校验/悬停 class 变化不会拆分一次连续输入 - - 输入事务使用浏览器源时间并保存全部 `raw_event_ids`，支持从原始日志审计和重算 - - 完整 Dump / Screenshot 由后台 Evidence Coordinator 合并执行；手动检查点仍保留稳定采集 -- **桌面**：UIA；Windows Recorder 使用进程外 `WH_MOUSE_LL` / `WH_KEYBOARD_LL` 事件监听，点击后由独立线程执行 `ControlFromPoint`。Hook 不注入目标进程；`GetAsyncKeyState` 仅作为 Hook 无法安装时的降级路径。 - - 输入使用焦点控件 Value/Text 变化聚合为一次 `fill`，不保存中文输入法的拼音按键；混合 DPI 应用会在键盘阶段重新探测正确坐标空间。 - - 已注册的桌面适配器会在宣布“录制就绪”前完成必要的 Accessibility warm；企业微信首次启动可能多等十几秒，但不会再让首个输入事务与后台暖树竞争。 - - 输入法候选/组合窗口只作为临时系统 UI 忽略，不会替换业务输入框；编辑器内的 Backspace/Delete 合并进最终 `fill`。 - - `Enter`、`Tab` 等功能键优先继承刚才的文本编辑器目标，不再退化为整个根窗口。 - - 鼠标按下时会枚举光标下可见顶层窗口，处理 owner window 吞掉浮窗命中的应用；后置树再把搜索/列表大容器提升到坐标命中的具体行。 - - 超过 5 秒的延迟证据不能用动态消息摘要/预览改写旧点击；坐标不可靠且只有一个可见结果行时，才使用唯一结果行作为保守回退。 - - 原始键盘活动只写入不含内容的事务标记；若发生输入但没有生成 `fill/type`，质量检查会明确告警。 -- **手机**：未实现，不得宣称已支持 - -## 输出目录（双写兼容） - -```text -_captures/_/ - flow.json # v1 schema；可选 v2 指针字段 - SUMMARY.md - action_flow.json # v2 - element_catalog.json # v2 - pages.json # Page / Document 记录 - journal/raw_events.jsonl - diagnostics/metrics.json - diagnostics/validation.json - diagnostics/replay_report.json # 静态、非破坏性的回放就绪预检 - steps/NN_xxx/ # v1：meta/tree/index/locatable/screen/SUMMARY - states// # v2：证据 - observations// # v2：每次真实采集证据（不去重） - actions/.json - elements/.json -``` - -旧读取方只认 `flow.json` + `steps/` 仍可工作。`flow.json` 中的 `v2` 字段可忽略。 - -### Schema 名 - -- `openclaw.ui_inspector.state.v1` -- `openclaw.ui_inspector.observation.v1` -- `openclaw.ui_inspector.action.v1` -- `openclaw.ui_inspector.element.v1` -- `openclaw.ui_inspector.element_catalog.v1` -- `openclaw.ui_inspector.action_flow.v1` -- `openclaw.ui_inspector.flow.v1`（原有） - -## 交给 AI 编程工具 - -优先提供： - -1. `action_flow.json` — 动作序列、目标和证据关联；异步证据不冒充精确 after state -2. `element_catalog.json` — logical_name、结构化 locator 候选、唯一性结果 -3. 需要排障时再附对应 `states//` 或旧 `steps/` 下的 tree/locatable/screen - -不要只丢一堆无动作语义的定时 dump。 - -`diagnostics.capture_validator` 会报告缺失的生产级定位器、目标元素引用、原始事件关联、根窗口按键目标、列表大容器点击及丢失的输入事务，避免“动作录到了”被误判为“可以稳定生成代码”。 - -## 当前能力边界 - -| 能力 | 状态 | -|------|------| -| click | 已实现（browser + windows） | -| fill/type 聚合 | 已实现 | -| press_key（Enter/Tab/Esc 等） | 已实现 | -| checkpoint | 已实现 | -| navigation | 部分（URL 变化观察） | -| 其余动作类型 | Schema 预留，`capability=schema_only` | -| 手机录制 | **未实现** | -| 截图像素级脱敏 | 未实现（敏感页在 meta 标记 `may_contain_sensitive_data`） | - -## 敏感信息 - -- 密码框 / token / cookie / 手机号 / 邮箱等会脱敏，不写入 action_flow / catalog 明文。 -- 截图无法可靠脱敏时标记风险；可用配置关闭敏感页截图保存。 -- **禁止**在测试或产物中使用真实客户数据。 - -## WeCom Adapter - -企业微信专用规则在 `adapters/wecom.py`： - -- `WeWorkWindow` → profile `wecom` -- `SearchResultWindow2` → profile `wecom_search` -- Warm **cache key = profile + hwnd**，主窗 warm 不会假装搜索浮层已暖 -- Recorder **捕获点击目标前不执行全量 warm**；warm 属于 dump/checkpoint/live 验证路径 - -## Legacy 间隔模式 - -Legacy 能力仅供底层兼容与开发排障，不在标准界面暴露。 -默认动作驱动模式下，heartbeat 只做输入聚合刷新，**相同 fingerprint 不新增 State，不产生 Action**。 - -## 冒烟与单测 - -```powershell -python tools/ui_inspector/smoke_check.py -python -m pytest tools/ui_inspector/tests -q -``` - -## 目录结构 - -```text -ui_inspector/ - core/ # 三端共用模型、管线、策略 - surfaces/ # browser / windows / future mobile - evidence/ # 异步证据协调 - locators/ # Locator Catalog 与稳定性 - storage/ # 新旧采集格式读写 - diagnostics/ # 指标与通用质量检查 - compatibility/ # v1/legacy 兼容入口 - ui/ # 新 UI facade；app.py 暂保留为 Tk 兼容入口 - adapters/ # 可选产品增强，不参与核心正确性 - tests/ # unit / contract / integration / e2e -``` - -`models/`、`recorder/`、`drivers/` 等旧路径暂时保留为兼容实现；新入口已从 `surfaces/`、`core/` 和 `storage/` 访问，后续可按模块逐步内收而不破坏现有技能。 - -暖树：`ensure_ui_ready`（kit≥1.2.6）；整页 dump：kit≥1.2.11。 diff --git a/tools/ui_inspector/__init__.py b/tools/ui_inspector/__init__.py deleted file mode 100644 index 08d010e..0000000 --- a/tools/ui_inspector/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""UI Inspector — development-only multi-surface capture (windows UIA + browser Playwright).""" - -__version__ = "0.16.0" diff --git a/tools/ui_inspector/__main__.py b/tools/ui_inspector/__main__.py deleted file mode 100644 index 1716458..0000000 --- a/tools/ui_inspector/__main__.py +++ /dev/null @@ -1,12 +0,0 @@ -"""python -m tools.ui_inspector (from skill-template root, if tools is a package) - -Prefer: python tools/ui_inspector/run.py -""" - -from __future__ import annotations - -import runpy -from pathlib import Path - -if __name__ == "__main__": - runpy.run_path(str(Path(__file__).resolve().parent / "app.py"), run_name="__main__") diff --git a/tools/ui_inspector/adapters/__init__.py b/tools/ui_inspector/adapters/__init__.py deleted file mode 100644 index 9c7deda..0000000 --- a/tools/ui_inspector/adapters/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""App adapters registry (WeCom-specific rules live here, not in generic layers).""" - -from .base import AppAdapter -from .registry import get_adapter, register_adapter -from . import wecom as _wecom # noqa: F401 — register on import - -__all__ = ["AppAdapter", "get_adapter", "register_adapter"] diff --git a/tools/ui_inspector/adapters/base.py b/tools/ui_inspector/adapters/base.py deleted file mode 100644 index 76d5793..0000000 --- a/tools/ui_inspector/adapters/base.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Adapter base protocol.""" - -from __future__ import annotations - -from typing import Any, Optional, Protocol - - -class AppAdapter(Protocol): - name: str - - def matches(self, *, class_name: str = "", title: str = "", process_name: str = "") -> bool: ... - - def wake_profile(self, *, class_name: str = "", title: str = "") -> Optional[str]: ... - - def cache_key(self, *, profile: str, hwnd: int = 0) -> str: ... - - def propose_postconditions(self, changes: list[dict[str, Any]]) -> list[dict[str, Any]]: ... diff --git a/tools/ui_inspector/adapters/registry.py b/tools/ui_inspector/adapters/registry.py deleted file mode 100644 index 5ce07f5..0000000 --- a/tools/ui_inspector/adapters/registry.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Adapter registry.""" - -from __future__ import annotations - -from typing import Any, Optional - -_REGISTRY: list[Any] = [] - - -def register_adapter(adapter: Any) -> Any: - _REGISTRY.append(adapter) - return adapter - - -def get_adapter( - *, - class_name: str = "", - title: str = "", - process_name: str = "", -) -> Optional[Any]: - for ad in _REGISTRY: - try: - if ad.matches(class_name=class_name, title=title, process_name=process_name): - return ad - except Exception: - continue - return None - - -def all_adapters() -> list[Any]: - return list(_REGISTRY) diff --git a/tools/ui_inspector/adapters/wecom.py b/tools/ui_inspector/adapters/wecom.py deleted file mode 100644 index 3fa2f53..0000000 --- a/tools/ui_inspector/adapters/wecom.py +++ /dev/null @@ -1,103 +0,0 @@ -"""WeCom (企业微信) adapter — profile / window / postcondition helpers.""" - -from __future__ import annotations - -from typing import Any, Optional - -from .registry import register_adapter - -PROFILE_WECOM = "wecom" -PROFILE_WECOM_SEARCH = "wecom_search" - -_THRESHOLDS = { - PROFILE_WECOM: 80, - PROFILE_WECOM_SEARCH: 20, -} - - -class WeComAdapter: - name = "wecom" - - def matches( - self, *, class_name: str = "", title: str = "", process_name: str = "" - ) -> bool: - return self.wake_profile(class_name=class_name, title=title) is not None or ( - "WXWork" in (process_name or "") - ) - - def wake_profile(self, *, class_name: str = "", title: str = "") -> Optional[str]: - cls = (class_name or "").strip() - title = title or "" - if cls == "SearchResultWindow2": - return PROFILE_WECOM_SEARCH - if cls == "WeWorkWindow": - return PROFILE_WECOM - if "WeWork" in cls and ( - "企业微信" in title or "WeCom" in title or "wecom" in title.lower() - ): - return PROFILE_WECOM - return None - - def cache_key(self, *, profile: str, hwnd: int = 0) -> str: - """Separate main window vs SearchResultWindow2 warm caches.""" - return f"{profile}:hwnd:{int(hwnd or 0)}" - - def warm_threshold(self, profile: str) -> int: - return int(_THRESHOLDS.get(profile, 80)) - - def is_search_popup(self, *, class_name: str = "") -> bool: - return (class_name or "") == "SearchResultWindow2" - - def is_main_window(self, *, class_name: str = "") -> bool: - return (class_name or "") == "WeWorkWindow" - - def known_ids(self) -> dict[str, str]: - return { - "searchedit": "search_box", - "groupbuddyname": "contact_row_name", - "search_name": "search_result_name", - "send_rich_edit": "compose_box", - "panel_title_label": "panel_title", - } - - def propose_postconditions(self, changes: list[dict[str, Any]]) -> list[dict[str, Any]]: - props: list[dict[str, Any]] = [] - for ch in changes or []: - kind = ch.get("kind") - detail = ch.get("detail") or {} - if kind == "popup_disappeared" and "SearchResult" in str( - detail.get("class_name") or "" - ): - props.append( - { - "description": "SearchResultWindow2 disappeared", - "kind": "window_absent", - "status": "proposed", - "detail": detail, - } - ) - if kind == "element_appeared": - name = str(detail.get("name") or detail.get("automation_id") or "") - if name in {"send_rich_edit", "main_chattext", "panel_title_label"}: - props.append( - { - "description": f"WeCom element {name} appeared", - "kind": "element_present", - "status": "proposed", - "detail": detail, - } - ) - if kind == "title_changed": - props.append( - { - "description": f"Chat title became {detail.get('to')!r}", - "kind": "title_equals", - "status": "proposed", - "detail": detail, - } - ) - return props - - -wecom_adapter = WeComAdapter() -register_adapter(wecom_adapter) diff --git a/tools/ui_inspector/app.py b/tools/ui_inspector/app.py deleted file mode 100644 index 67f4611..0000000 --- a/tools/ui_inspector/app.py +++ /dev/null @@ -1,2001 +0,0 @@ -"""Tk panel v0.14: Action-driven UI Automation Recorder (compatible with v1 dumps). - -拾取模式（单选，记忆上次）：浏览器 / 桌面 / 手机。 -- 浏览器：强制 Playwright；起始 URL 自动打开；Browser Recorder -- 桌面：强制 UIA；Windows Recorder（进程外轮询，无注入） -- 手机：占位未实现 - -录制模式：自动录制（默认）/ 引导录制 / 手动检查点；legacy 间隔 dump 可选。 -""" - -from __future__ import annotations - -import argparse -import ctypes -import os -import queue -import subprocess -import sys -import threading -from ctypes import wintypes -from pathlib import Path -import tkinter as tk -from tkinter import messagebox, simpledialog, ttk - -_HERE = Path(__file__).resolve().parent -if str(_HERE) not in sys.path: - sys.path.insert(0, str(_HERE)) - -import artifacts as art # noqa: E402 -import desktop_uia as uia # noqa: E402 -import kit_browser # noqa: E402 -import skip_policy # noqa: E402 -import target # noqa: E402 -import user_prefs # noqa: E402 -from surfaces.registry import get_driver # noqa: E402 -from flow_session import FlowSession # noqa: E402 -from drivers.base import DumpResult # noqa: E402 - -try: - from . import __version__ -except Exception: - try: - import importlib.util - - _spec = importlib.util.spec_from_file_location( - "ui_inspector_pkg", _HERE / "__init__.py" - ) - _mod = importlib.util.module_from_spec(_spec) # type: ignore[arg-type] - assert _spec and _spec.loader - _spec.loader.exec_module(_mod) - __version__ = getattr(_mod, "__version__", "0.16.0") - except Exception: - __version__ = "0.16.0" - -# Legacy interval mode: after each dump finishes, wait this long, then dump again. -_AUTO_GAP_MS = 2000 -# Heartbeat for action-driven mode (flush idle input; do NOT create steps). -_HEARTBEAT_MS = 800 -# After frontmost target is resolved, short delay before first dump. -_AUTO_START_DELAY_MS = 400 -# After「开始流程」with auto-pick: countdown so user can focus the target app. -_START_COUNTDOWN_SEC = 5 -_FRONTMOST_WAIT_SEC = 15.0 -_STABILITY_TIMEOUT_SEC = 3.0 - -# Recording modes -_MODE_AUTO = "auto" -_MODE_GUIDED = "guided" -_MODE_CHECKPOINT = "checkpoint" -_MODE_LEGACY = "legacy_interval" - -_WINDOW_RIGHT_MARGIN = 16 -_WINDOW_BOTTOM_MARGIN = 16 -_WINDOW_MIN_WIDTH = 520 -_WINDOW_MIN_HEIGHT = 600 - -# Default flow-name slugs aligned with the pick-mode radio labels. -# Switching mode only auto-updates the field when it still holds one of these. -_MODE_FLOW_NAMES = { - target.SURFACE_BROWSER: "browser", - target.SURFACE_WINDOWS: "desktop", - target.SURFACE_ANDROID: "mobile", -} -_DEFAULT_FLOW_NAME_SLUGS = frozenset(_MODE_FLOW_NAMES.values()) - - -def _flow_name_for_mode(mode: str) -> str: - return _MODE_FLOW_NAMES.get(mode, "desktop") - - -def _clamp(value: int, low: int, high: int) -> int: - return max(low, min(high, value)) - - -def _default_window_geometry( - work_area: tuple[int, int, int, int], - *, - side: str = "right", -) -> tuple[int, int, int, int]: - """Return a narrow console geometry inside a monitor work area. - - The top gap deliberately leaves the target application's title-bar controls - unobstructed. Coordinates may be negative for monitors left of the primary. - """ - - left, top, right, bottom = work_area - work_width = max(1, right - left) - work_height = max(1, bottom - top) - width = _clamp(round(work_width * 0.30), 560, 620) - width = min(width, max(420, work_width - 32)) - top_gap = _clamp(round(work_height * 0.07), 64, 88) - height = max(480, work_height - top_gap - _WINDOW_BOTTOM_MARGIN) - x = ( - left + _WINDOW_RIGHT_MARGIN - if side == "left" - else right - width - _WINDOW_RIGHT_MARGIN - ) - y = top + top_gap - return width, height, x, y - - -def _format_geometry(width: int, height: int, x: int, y: int) -> str: - x_part = f"+{x}" if x >= 0 else str(x) - y_part = f"+{y}" if y >= 0 else str(y) - return f"{width}x{height}{x_part}{y_part}" - - -def _format_live_action(item: dict) -> str: - """Compact, surface-neutral live action text for the result panel.""" - labels = { - "click": "点击", "fill": "输入", "type": "输入", - "press_key": "按键", "navigation": "导航", "checkpoint": "检查点", - } - phases = { - "captured": "已捕获，证据采集中", - "evidence": "证据已更新", - "evidence_failed": "证据采集失败", - } - action_type = str(item.get("action_type") or "unknown") - lines = [ - f"#{item.get('sequence')} {labels.get(action_type, action_type)} · " - f"{phases.get(str(item.get('phase') or ''), str(item.get('phase') or ''))}", - f" 表面: {item.get('surface') or '-'} 目标: " - f"{item.get('target_name') or '-'} ({item.get('target_type') or '-'})", - ] - if item.get("input_summary"): - lines.append(f" 输入: {item['input_summary']}") - if item.get("window_class"): - lines.append( - f" 窗口: {item.get('window_title') or '-'} [{item.get('window_class')}]" - ) - if item.get("locator"): - lines.append( - f" 定位: {item['locator']} match={item.get('match_count')} " - f"unique={item.get('unique')}" - ) - elif item.get("phase") == "evidence_failed": - lines.append(f" 错误: {item.get('error') or '未知错误'}") - else: - lines.append(" 定位: 等待异步证据") - return "\n".join(lines) - - -def _merge_current_live_action(current: dict | None, incoming: dict) -> dict | None: - """Keep the newest action while allowing its evidence phase to replace it.""" - if not str(incoming.get("id") or ""): - return current - if current is None: - return dict(incoming) - current_sequence = int(current.get("sequence") or 0) - incoming_sequence = int(incoming.get("sequence") or 0) - if incoming_sequence < current_sequence: - return current - return dict(incoming) - - -class InspectorApp: - def __init__( - self, - *, - captures_root: Path, - scene: str = "", - max_depth: int = 16, - max_nodes: int = 8000, - ) -> None: - self.captures_root = captures_root - self.scene = scene - self.max_depth = max_depth - self.max_nodes = max_nodes - - self.flow_dir: Path | None = None - self.flow: dict | None = None - self.last_window: dict | None = None - self.sticky_pid: int | None = None - # Browser entries use pid; native entries use (pid, window class) so a - # WeCom SearchResultWindow2 is not mistaken for its already-warmed main - # WeWorkWindow merely because both belong to the same process. - self._warmed_pids: set[object] = set() - self._busy = False - self._auto_loop_active = False - self._pending_end = False - self._auto_after_id: str | None = None - self._countdown_after_id: str | None = None - self._countdown_left = 0 - self._session: FlowSession | None = None - self._desktop_entry_dump: DumpResult | None = None - self._heartbeat_after_id: str | None = None - self._rec_paused = False - self._current_live_action: dict | None = None - self._ui_queue: queue.Queue[tuple[object, tuple, dict]] = queue.Queue() - self._ui_poll_after_id: str | None = None - self._closed = False - - prefs = user_prefs.load_prefs() - initial_mode = str(prefs.get("mode") or user_prefs.DEFAULT_MODE) - if initial_mode in {"mobile", "phone", "ios"}: - initial_mode = target.SURFACE_ANDROID - if initial_mode not in { - target.SURFACE_BROWSER, - target.SURFACE_WINDOWS, - target.SURFACE_ANDROID, - }: - initial_mode = user_prefs.DEFAULT_MODE - scene_name = (scene or "").strip() - initial_flow_name = scene_name or _flow_name_for_mode(initial_mode) - - self.root = tk.Tk() - self.root.title(f"OpenClaw UI Inspector v{__version__}") - self._dock_side = "right" - self._apply_default_geometry() - self.root.attributes("-topmost", True) - - self.flow_name_var = tk.StringVar(value=initial_flow_name) - self.mode_var = tk.StringVar(value=initial_mode) - self.start_url_var = tk.StringVar(value=str(prefs.get("start_url") or "")) - # Standard UI always records actions automatically. Legacy strategies - # remain supported internally for old captures and developer tooling. - self.auto_pick_var = tk.BooleanVar(value=True) - self.recording_mode_var = tk.StringVar(value=_MODE_AUTO) - # 默认开：悬停即可采搜索浮层，不必点进会话；均可随时取消勾选回退旧行为 - self.under_cursor_var = tk.BooleanVar(value=True) - self.prefer_wecom_search_var = tk.BooleanVar(value=True) - self.flow_var = tk.StringVar(value="流程：未开始（请先点「开始流程」）") - self.target_var = tk.StringVar(value="等待开始流程") - self.rec_stats_var = tk.StringVar( - value="recorder: idle | actions=0 states=0 observations=0 elements=0" - ) - self.status_var = tk.StringVar( - value="选择模式后点击「开始流程」。" - ) - - self._build_ui() - for var in ( - self.flow_var, - self.rec_stats_var, - self.target_var, - self.status_var, - ): - var.trace_add("write", self._on_status_var_changed) - self._render_status_summary() - self._refresh_mode_widgets() - self.root.protocol("WM_DELETE_WINDOW", self._on_close) - self._schedule_ui_queue_poll() - - def _current_monitor_work_area(self) -> tuple[int, int, int, int]: - """Get the work area of the monitor containing the mouse pointer.""" - - if sys.platform == "win32": - try: - class MONITORINFO(ctypes.Structure): - _fields_ = [ - ("cbSize", wintypes.DWORD), - ("rcMonitor", wintypes.RECT), - ("rcWork", wintypes.RECT), - ("dwFlags", wintypes.DWORD), - ] - - point = wintypes.POINT() - user32 = ctypes.windll.user32 - user32.GetCursorPos.argtypes = [ctypes.POINTER(wintypes.POINT)] - user32.GetCursorPos.restype = wintypes.BOOL - user32.MonitorFromPoint.argtypes = [wintypes.POINT, wintypes.DWORD] - user32.MonitorFromPoint.restype = wintypes.HANDLE - user32.GetMonitorInfoW.argtypes = [ - wintypes.HANDLE, - ctypes.POINTER(MONITORINFO), - ] - user32.GetMonitorInfoW.restype = wintypes.BOOL - if user32.GetCursorPos(ctypes.byref(point)): - monitor = user32.MonitorFromPoint(point, 2) - info = MONITORINFO() - info.cbSize = ctypes.sizeof(info) - if monitor and user32.GetMonitorInfoW(monitor, ctypes.byref(info)): - area = info.rcWork - return area.left, area.top, area.right, area.bottom - except Exception: - pass - return 0, 0, self.root.winfo_screenwidth(), self.root.winfo_screenheight() - - def _apply_default_geometry(self) -> None: - width, height, x, y = _default_window_geometry( - self._current_monitor_work_area(), side=self._dock_side - ) - self.root.geometry(_format_geometry(width, height, x, y)) - self.root.minsize(min(_WINDOW_MIN_WIDTH, width), min(_WINDOW_MIN_HEIGHT, height)) - - def _toggle_dock(self) -> None: - self._dock_side = "left" if self._dock_side == "right" else "right" - self._apply_default_geometry() - self.dock_btn.configure(text="靠右" if self._dock_side == "left" else "靠左") - - def _on_status_var_changed(self, *_args: object) -> None: - self._render_status_summary() - - def _render_status_summary(self) -> None: - if not hasattr(self, "status_text"): - return - if not self.flow: - headline = "● 未开始 · recorder idle" - else: - v2 = self.flow.get("v2") or {} - live = self._session.stats() if self._session else {} - status = str(self.flow.get("status") or "unknown") - status_name = {"open": "录制中", "ended": "已结束"}.get(status, status) - action_count = live.get("action_count", v2.get("action_count", 0)) - state_count = live.get("state_count", v2.get("state_count", 0)) - dropped = live.get("dropped_events", 0) - input_health = live.get("input_backend") or {} - input_suffix = "" - if input_health: - input_suffix = ( - f" · input {input_health.get('raw_input_events', 0)}" - f" · {input_health.get('backend', 'unknown')}" - ) - headline = ( - f"● {status_name} · actions {action_count} · states {state_count}" - f" · dropped {dropped}{input_suffix}" - ) - target_line = self.target_var.get().strip() - status_line = self.status_var.get().strip() - lines = [headline] - if target_line: - lines.append(target_line) - if status_line and status_line != target_line: - lines.append(status_line) - self.status_text.configure(state=tk.NORMAL) - self.status_text.delete("1.0", tk.END) - self.status_text.insert(tk.END, "\n".join(line for line in lines if line)) - self.status_text.configure(state=tk.DISABLED) - - def _post_ui(self, callback, *args, **kwargs) -> None: - """Worker-safe UI handoff; workers must never call Tk directly.""" - self._ui_queue.put((callback, args, kwargs)) - - def _schedule_ui_queue_poll(self) -> None: - self._ui_poll_after_id = self.root.after(40, self._drain_ui_queue) - - def _drain_ui_queue(self) -> None: - self._ui_poll_after_id = None - for _ in range(100): - try: - callback, args, kwargs = self._ui_queue.get_nowait() - except queue.Empty: - break - try: - callback(*args, **kwargs) # type: ignore[operator] - except Exception as exc: - try: - self._set_status(f"后台结果处理失败: {exc}") - except Exception: - pass - try: - if self.root.winfo_exists(): - self._schedule_ui_queue_poll() - except Exception: - pass - - def _current_mode(self) -> str: - mode = (self.mode_var.get() or user_prefs.DEFAULT_MODE).strip().lower() - if mode in {"mobile", "phone", "ios"}: - return target.SURFACE_ANDROID - if mode not in { - target.SURFACE_BROWSER, - target.SURFACE_WINDOWS, - target.SURFACE_ANDROID, - }: - return user_prefs.DEFAULT_MODE - return mode - - def _persist_prefs(self) -> None: - user_prefs.save_prefs( - mode=self._current_mode(), - start_url=(self.start_url_var.get() or "").strip(), - recording_mode=_MODE_AUTO, - ) - - def _current_recording_mode(self) -> str: - return _MODE_AUTO - - def _is_legacy_interval(self) -> bool: - return self._current_recording_mode() == _MODE_LEGACY - - def _on_mode_change(self) -> None: - self._persist_prefs() - self._sync_default_flow_name() - self._refresh_mode_widgets() - mode = self._current_mode() - if mode == target.SURFACE_BROWSER: - self._set_status( - "已选「浏览器自动化」：开始流程后将打开起始 URL 并 dump Playwright 页。" - ) - elif mode == target.SURFACE_WINDOWS: - self._set_status( - "已选「桌面自动化」：开始流程后按光标/前台窗口做 UIA dump（不启浏览器）。" - ) - else: - self._set_status("已选「手机自动化」：尚未实现，开始流程时会提示。") - - def _sync_default_flow_name(self) -> None: - """Keep flow name aligned with pick mode unless the user customized it.""" - current = (self.flow_name_var.get() or "").strip() - if current and current not in _DEFAULT_FLOW_NAME_SLUGS: - return - self.flow_name_var.set(_flow_name_for_mode(self._current_mode())) - - def _refresh_mode_widgets(self) -> None: - mode = self._current_mode() - is_browser = mode == target.SURFACE_BROWSER - is_desktop = mode == target.SURFACE_WINDOWS - try: - if is_browser: - self.url_row.grid() - else: - self.url_row.grid_remove() - except Exception: - pass - try: - if is_desktop: - self.desktop_options.pack( - fill=tk.X, - padx=8, - pady=(0, 4), - before=self.result_panel, - ) - else: - self.desktop_options.pack_forget() - except Exception: - pass - - def _synthetic_browser_info(self) -> uia.WindowInfo: - return uia.WindowInfo( - hwnd=0, - pid=0, - title="Playwright", - class_name="PlaywrightPage", - process_name="chrome.exe", - ) - - def _build_ui(self) -> None: - pad = {"padx": 8, "pady": (4, 2)} - - top = ttk.Frame(self.root) - top.pack(fill=tk.X, padx=8, pady=(8, 2)) - top.columnconfigure(1, weight=1) - ttk.Label(top, text="流程名").grid(row=0, column=0, sticky=tk.W) - ttk.Entry(top, textvariable=self.flow_name_var).grid( - row=0, column=1, sticky=tk.EW, padx=(4, 6) - ) - ttk.Button(top, text="打开目录", command=self.on_open_out).grid( - row=0, column=2, padx=(0, 4) - ) - self.dock_btn = ttk.Button(top, text="靠左", width=6, command=self._toggle_dock) - self.dock_btn.grid(row=0, column=3) - - mode_box = ttk.LabelFrame(self.root, text="拾取模式") - mode_box.pack(fill=tk.X, **pad) - mode_box.columnconfigure(0, weight=1) - mode_row = ttk.Frame(mode_box) - mode_row.grid(row=0, column=0, sticky=tk.EW, padx=6, pady=(2, 0)) - for column in range(3): - mode_row.columnconfigure(column, weight=1) - ttk.Radiobutton( - mode_row, - text="浏览器自动化", - value=target.SURFACE_BROWSER, - variable=self.mode_var, - command=self._on_mode_change, - ).grid(row=0, column=0, sticky=tk.W, padx=4) - ttk.Radiobutton( - mode_row, - text="桌面自动化", - value=target.SURFACE_WINDOWS, - variable=self.mode_var, - command=self._on_mode_change, - ).grid(row=0, column=1, sticky=tk.W, padx=4) - ttk.Radiobutton( - mode_row, - text="手机自动化", - value=target.SURFACE_ANDROID, - variable=self.mode_var, - command=self._on_mode_change, - ).grid(row=0, column=2, sticky=tk.W, padx=4) - - self.url_row = ttk.Frame(mode_box) - self.url_row.grid(row=1, column=0, sticky=tk.EW, padx=6, pady=(2, 4)) - self.url_row.columnconfigure(1, weight=1) - ttk.Label(self.url_row, text="起始 URL").grid(row=0, column=0, sticky=tk.W) - ttk.Entry(self.url_row, textvariable=self.start_url_var).grid( - row=0, column=1, sticky=tk.EW, padx=(6, 0) - ) - - btns = ttk.LabelFrame(self.root, text="流程控制") - btns.pack(fill=tk.X, **pad) - for column in range(3): - btns.columnconfigure(column, weight=1, uniform="flow_buttons") - self.start_btn = ttk.Button(btns, text="开始", command=self.on_start_flow) - self.start_btn.grid(row=0, column=0, sticky=tk.EW, padx=(6, 2), pady=(3, 6)) - self.pause_btn = ttk.Button(btns, text="暂停", command=self.on_pause_resume) - self.pause_btn.grid(row=0, column=1, sticky=tk.EW, padx=2, pady=(3, 6)) - self.end_btn = ttk.Button(btns, text="结束", command=self.on_end_flow) - self.end_btn.grid(row=0, column=2, sticky=tk.EW, padx=(2, 6), pady=(3, 6)) - - self.desktop_options = ttk.LabelFrame(self.root, text="桌面拾取选项") - self.under_cursor_chk = ttk.Checkbutton( - self.desktop_options, - text="光标下优先（悬停、不点击）", - variable=self.under_cursor_var, - ) - self.under_cursor_chk.pack(anchor=tk.W, padx=8, pady=(2, 0)) - self.wecom_chk = ttk.Checkbutton( - self.desktop_options, - text="企微搜索浮层优先（SearchResultWindow2）", - variable=self.prefer_wecom_search_var, - ) - self.wecom_chk.pack(anchor=tk.W, padx=8, pady=(0, 4)) - - self.result_panel = ttk.LabelFrame(self.root, text="运行状态与拾取结果") - self.result_panel.pack(fill=tk.BOTH, expand=True, padx=8, pady=(4, 8)) - - self.status_text = tk.Text( - self.result_panel, - height=3, - wrap=tk.NONE, - borderwidth=0, - padx=5, - pady=3, - ) - self.status_text.pack(fill=tk.X, padx=5, pady=(3, 2)) - self.status_text.configure(state=tk.DISABLED) - ttk.Separator(self.result_panel, orient=tk.HORIZONTAL).pack( - fill=tk.X, padx=5, pady=(0, 3) - ) - - detail_box = ttk.Frame(self.result_panel) - detail_box.pack(fill=tk.BOTH, expand=True, padx=3, pady=(0, 3)) - scroll = ttk.Scrollbar(detail_box) - scroll.pack(side=tk.RIGHT, fill=tk.Y) - self.detail = tk.Text( - detail_box, - height=10, - wrap=tk.WORD, - borderwidth=0, - yscrollcommand=scroll.set, - ) - self.detail.pack(fill=tk.BOTH, expand=True, padx=(3, 0), pady=3) - scroll.config(command=self.detail.yview) - self.detail.insert( - tk.END, - "1. 选择拾取模式\n" - "2. 点击「开始流程」，然后操作目标窗口\n" - "3. 完成后点击「结束流程」\n\n" - "动作、元素定位和诊断结果会显示在这里。\n", - ) - self.detail.configure(state=tk.DISABLED) - self._refresh_button_states() - - def _flow_is_open(self) -> bool: - return bool(self.flow_dir and self.flow and self.flow.get("status") == "open") - - def _in_start_countdown(self) -> bool: - return self._countdown_after_id is not None or self._countdown_left > 0 - - def _on_recording_mode_change(self) -> None: - self._persist_prefs() - mode = self._current_recording_mode() - if mode == _MODE_LEGACY: - self._set_status("已选 Legacy 间隔 dump：将按约 2 秒串行写 steps（兼容旧行为）。") - elif mode == _MODE_GUIDED: - self._set_status("已选引导录制：每捕获一个动作后在本框展示定位与后置条件建议。") - elif mode == _MODE_CHECKPOINT: - self._set_status("已选手动检查点：请用「采集界面检查点」；不会自动录用户操作。") - self.auto_pick_var.set(False) - else: - self._set_status("已选自动录制：Recorder 捕获用户操作并双写 v1/v2 资产。") - - def _refresh_button_states(self) -> None: - open_flow = self._flow_is_open() - counting = self._in_start_countdown() - self.end_btn.configure(state=tk.NORMAL if open_flow else tk.DISABLED) - self.start_btn.configure( - state=tk.DISABLED if (self._busy or counting or open_flow) else tk.NORMAL - ) - try: - self.pause_btn.configure( - state=( - tk.NORMAL - if (open_flow and self._session and not self._busy and not counting) - else tk.DISABLED - ) - ) - except Exception: - pass - - def _update_rec_stats(self) -> None: - if not self._session: - self.rec_stats_var.set( - "recorder: idle | actions=0 states=0 observations=0 elements=0" - ) - return - st = self._session.stats() - input_health = st.get("input_backend") or {} - input_summary = "" - if input_health: - input_summary = ( - f" input={input_health.get('backend', 'unknown')}" - f" raw={input_health.get('raw_input_events', 0)}" - f" mouse={input_health.get('mouse_events', 0)}" - f" keyboard={input_health.get('keyboard_events', 0)}" - f" input_dropped={input_health.get('queue_dropped', 0)}" - f" input_error={input_health.get('last_error') or '-'} |" - ) - self.rec_stats_var.set( - f"recorder={st.get('recorder_status')}/{st.get('surface_recorder')} | " - f"actions={st.get('action_count')} states={st.get('state_count')} " - f"observations={st.get('observation_count')} elements={st.get('element_count')} " - f"queue={st.get('queued_events', 0)} dropped={st.get('dropped_events', 0)} | " - f"{input_summary} " - f"last={st.get('last_action_type') or '-'} " - f"target={st.get('last_target') or '-'} " - f"page={st.get('last_page_id') or '-'} " - f"obs={st.get('last_observation_id') or '-'} " - f"match={st.get('last_match_count')} unique={st.get('last_unique')}" - ) - - def _update_flow_label(self) -> None: - if not self.flow_dir or not self.flow: - self.flow_var.set("流程：未开始（请先点「开始流程」）") - return - if self._auto_loop_active: - auto = "LOOP" if self._is_legacy_interval() else "REC" - elif self.auto_pick_var.get() and self._flow_is_open(): - auto = "armed" - else: - auto = "off" - v2 = (self.flow or {}).get("v2") or {} - live = self._session.stats() if self._session else {} - action_count = live.get("action_count", v2.get("action_count", 0)) - state_count = live.get("state_count", v2.get("state_count", 0)) - observation_count = live.get( - "observation_count", v2.get("observation_count", 0) - ) - self.flow_var.set( - f"流程：{self.flow.get('flow_name')} | mode={self._current_mode()} | " - f"rec={self._current_recording_mode()} | " - f"status={self.flow.get('status')} | " - f"legacy_steps={self.flow.get('step_count')} " - f"actions={action_count} states={state_count} " - f"observations={observation_count} | auto={auto} | {self.flow_dir}" - ) - self._update_rec_stats() - - def _set_status(self, text: str) -> None: - self.status_var.set(text) - - def _set_detail(self, text: str) -> None: - self.detail.configure(state=tk.NORMAL) - self.detail.delete("1.0", tk.END) - self.detail.insert(tk.END, text) - self.detail.see("1.0") - self.detail.configure(state=tk.DISABLED) - - def _on_session_action_update(self, action, phase: str) -> None: - """Snapshot an ActionRecord on a worker, then hand it safely to Tk.""" - target_snapshot = dict(getattr(action, "target_snapshot", {}) or {}) - evidence = dict(getattr(action, "evidence", {}) or {}) - input_payload = dict(getattr(action, "input", {}) or {}) - scope = dict(target_snapshot.get("window_scope") or {}) - input_summary = "" - if input_payload.get("key"): - input_summary = str(input_payload.get("key")) - elif input_payload.get("text"): - value = str(input_payload.get("text")) - input_summary = value if len(value) <= 80 else value[:77] + "..." - item = { - "id": str(getattr(action, "id", "") or ""), - "sequence": int(getattr(action, "sequence", 0) or 0), - "action_type": str(getattr(action, "action_type", "") or ""), - "surface": str(getattr(action, "surface", "") or ""), - "phase": phase, - "target_name": target_snapshot.get("name") - or target_snapshot.get("automation_id") - or target_snapshot.get("semantic_name") - or target_snapshot.get("role") - or target_snapshot.get("tag") - or "-", - "target_type": target_snapshot.get("control_type") - or target_snapshot.get("role") - or target_snapshot.get("tag") - or "-", - "window_title": scope.get("title") or "", - "window_class": scope.get("class_name") or "", - "input_summary": input_summary, - "locator": evidence.get("primary_locator") or "", - "match_count": evidence.get("locator_match_count"), - "unique": evidence.get("locator_unique"), - "error": evidence.get("eventual_evidence_error") - or evidence.get("error") - or "", - } - self._post_ui(self._apply_live_action_update, item) - - def _apply_live_action_update(self, item: dict) -> None: - updated = _merge_current_live_action(self._current_live_action, item) - if updated is self._current_live_action: - return - self._current_live_action = updated - assert updated is not None - self.target_var.set( - f"当前动作：#{updated.get('sequence')} {updated.get('action_type')} · " - f"{updated.get('phase')}" - ) - self._set_detail("当前步骤\n\n" + _format_live_action(updated)) - - def _show_picking_placeholder(self, *, auto: bool) -> None: - n = int((self.flow or {}).get("step_count") or 0) + 1 - mode = "自动" if auto else "检查点" - self.target_var.set(f"{mode}采集中…（即将写入第 {n} 步）") - self._set_status(f"正在{mode}采集：已清空本框上次结果，请稍候…") - self._set_detail( - "（已清空上次本步结果）\n\n" - f"正在读取控件树，完成后写入当前流程第 {n} 步…\n" - ) - - def _exclude_hwnds(self) -> set[int]: - try: - return {int(self.root.winfo_id())} - except Exception: - return set() - - def _cancel_auto_timer(self) -> None: - if self._auto_after_id is not None: - try: - self.root.after_cancel(self._auto_after_id) - except Exception: - pass - self._auto_after_id = None - - def _cancel_countdown(self) -> None: - if self._countdown_after_id is not None: - try: - self.root.after_cancel(self._countdown_after_id) - except Exception: - pass - self._countdown_after_id = None - self._countdown_left = 0 - - def _cancel_heartbeat(self) -> None: - if self._heartbeat_after_id is not None: - try: - self.root.after_cancel(self._heartbeat_after_id) - except Exception: - pass - self._heartbeat_after_id = None - - def _stop_auto_loop(self) -> None: - self._auto_loop_active = False - self._cancel_auto_timer() - self._cancel_countdown() - self._cancel_heartbeat() - - def _stop_session(self) -> None: - if self._session is not None: - try: - self._session.stop() - except Exception: - pass - self._session = None - self._rec_paused = False - try: - self.pause_btn.configure(text="暂停") - except Exception: - pass - - def _schedule_heartbeat(self) -> None: - self._cancel_heartbeat() - if not self._auto_loop_active or not self._flow_is_open(): - return - if self._is_legacy_interval(): - return - self._heartbeat_after_id = self.root.after(_HEARTBEAT_MS, self._heartbeat_tick) - - def _heartbeat_tick(self) -> None: - self._heartbeat_after_id = None - if not self._auto_loop_active or not self._flow_is_open(): - return - if self._session and not self._rec_paused: - try: - self._session.heartbeat() - except Exception: - pass - self._update_flow_label() - # Guided mode: refresh last action detail - if self._current_recording_mode() == _MODE_GUIDED: - self._show_guided_last_action() - self._schedule_heartbeat() - - def _show_guided_last_action(self) -> None: - if not self._session: - return - act = self._session.manager.last_action - if not act: - return - lines = [ - f"最近动作 #{act.sequence} {act.action_type} status={act.status}", - f"target: {(act.target_snapshot or {}).get('name') or (act.target_snapshot or {}).get('automation_id')}", - f"scope: {(act.target_snapshot or {}).get('window_scope')}", - f"primary locator: {(act.evidence or {}).get('primary_locator')}", - f"match count: {(act.evidence or {}).get('locator_match_count')} unique={(act.evidence or {}).get('locator_unique')}", - "proposed postconditions:", - ] - for p in (act.proposed_postconditions or [])[:8]: - lines.append(f" - [{p.get('status')}] {p.get('description')}") - if self._session.manager.last_warning: - lines.append(f"warning: {self._session.manager.last_warning}") - self._set_detail("\n".join(lines)) - - def _make_dump_fn(self): - """Build a dump callable for SnapshotService (runs on worker threads).""" - - # Snapshot every Tk-owned value on the main thread. The returned - # callable is deliberately free of Tk calls so it cannot participate - # in a Tk <-> recorder lock inversion. - mode = self._current_mode() - under_cursor = bool(self.under_cursor_var.get()) - prefer_search = bool(self.prefer_wecom_search_var.get()) - sticky_pid = self.sticky_pid - exclude_hwnds = self._exclude_hwnds() - max_depth = self.max_depth - max_nodes = self.max_nodes - warmed_pids = self._warmed_pids - - def dump_fn( - *, page_id: str = "", document_id: str = "", frame_id: str = "", - target_hwnd: int = 0, target_pid: int = 0, - target_class_name: str = "", target_title: str = "", - ): - uia.ensure_com_sta() - surface = mode - sel_meta: dict = {"pick_mode": mode, "sticky_pid": sticky_pid} - if surface == target.SURFACE_BROWSER: - info = self._synthetic_browser_info() - sel_meta["strategy"] = "playwright_session" - elif target_hwnd: - try: - info = uia.window_info_from_hwnd(int(target_hwnd)) - except Exception: - info = None - if info is None and target_class_name == "SearchResultWindow2": - info = uia.find_wecom_search_result_window( - pid=int(target_pid or 0) or None, - exclude_hwnds=exclude_hwnds, - ) - if info is None: - raise RuntimeError( - f"event window disappeared: hwnd={target_hwnd} class={target_class_name}" - ) - cands = [info] - sel_meta.update( - { - "strategy": "event_window", - "chosen_hwnd": int(info.hwnd), - "chosen_class": info.class_name, - "event_title": target_title, - } - ) - sel_meta["candidate_count"] = 1 - else: - info, cands, win_meta = uia.resolve_capture_target( - exclude_hwnds=exclude_hwnds, - prefer_pid=sticky_pid, - under_cursor=under_cursor, - prefer_wecom_search_popup=prefer_search, - ) - sel_meta.update(win_meta) - sel_meta["candidate_count"] = len(cands) - driver = get_driver(surface) - warm_key = ( - (int(info.pid), str(info.class_name or "")) - if surface == target.SURFACE_WINDOWS - else int(info.pid or 0) - ) - warmup_mode = "light" if warm_key in warmed_pids else "full" - if surface == target.SURFACE_WINDOWS: - # Recording already has a persisted event-time element atlas. - # Evidence may probe the full tree, but must never run a - # blocking product wake in the middle of user input. - warmup_mode = "probe" - if target_class_name == "SearchResultWindow2": - # A search popup may live for only a few seconds. Persist its - # bounded tree immediately; heavyweight kit wake/FindAll remains - # supplementary and must not make the event-time snapshot miss it. - warmup_mode = "off" - # Recorder path: after-action dumps may warm; click capture itself never warms. - dump_kwargs = dict( - info=info, - selection_meta=sel_meta, - max_depth=(12 if target_class_name == "SearchResultWindow2" else max_depth), - max_nodes=(1500 if target_class_name == "SearchResultWindow2" else max_nodes), - warmup_mode=warmup_mode, - warmed_pids=warmed_pids, - ) - if surface == target.SURFACE_BROWSER: - dump_kwargs.update( - page_id=page_id, - document_id=document_id, - frame_id=frame_id, - ) - result = driver.dump(**dump_kwargs) - if info.pid: - warmed_pids.add(warm_key) - return result - - return dump_fn - - def _start_recording_session( - self, *, entry_dump=None, checkpoint_only: bool = False - ) -> None: - if not self.flow_dir or not self.flow: - return - rec_mode = _MODE_CHECKPOINT if checkpoint_only else self._current_recording_mode() - session = FlowSession( - self.flow_dir, - flow_name=str(self.flow.get("flow_name") or "flow"), - surface=self._current_mode(), - recording_mode=rec_mode, - dump_fn=self._make_dump_fn(), - exclude_hwnds=self._exclude_hwnds, - on_action_update=self._on_session_action_update, - stability_timeout_sec=_STABILITY_TIMEOUT_SEC, - ) - self._session = session - if rec_mode == _MODE_LEGACY: - self._auto_loop_active = True - self._schedule_auto_next(_AUTO_START_DELAY_MS) - return - - self._busy = True - self._refresh_button_states() - self._set_status("正在后台采集入口状态，界面仍可响应……") - - def work() -> None: - try: - if rec_mode == _MODE_CHECKPOINT: - session.manager.action_flow.recording_mode = _MODE_CHECKPOINT - session.manager.start(entry_dump=entry_dump) - session._refresh_stats() - else: - session.start(entry_dump=entry_dump) - stats = session.stats() - if int(stats.get("state_count") or 0) == 0: - warning = str(stats.get("warning") or "入口状态采集没有产物") - raise RuntimeError(f"ERROR:ENTRY_CAPTURE_FAILED {warning}") - except Exception as exc: - self._post_ui(self._on_recording_session_started, session, rec_mode, exc) - try: - session.stop() - except Exception: - pass - return - self._post_ui(self._on_recording_session_started, session, rec_mode, None) - - threading.Thread( - target=work, name="ui-inspector-session-start", daemon=True - ).start() - - def _on_recording_session_started( - self, session: FlowSession, rec_mode: str, error: BaseException | None - ) -> None: - if self._session is not session: - return - self._busy = False - if error is not None: - self._session = None - self._set_status(f"录制会话启动失败: {error}") - self._refresh_button_states() - return - self._auto_loop_active = True - self._rec_paused = False - self._update_flow_label() - self._refresh_button_states() - if rec_mode == _MODE_CHECKPOINT: - self._auto_loop_active = False - self._set_status("入口状态已采集；可手动添加界面检查点。") - else: - self.target_var.set("动作录制中：等待用户操作") - self._set_status("入口状态已采集，动作录制已就绪。") - self._schedule_heartbeat() - - def _start_auto_loop(self, *, first_delay_ms: int = _AUTO_START_DELAY_MS) -> None: - """Legacy interval loop OR action-driven recording start.""" - self._pending_end = False - if self._is_legacy_interval(): - self._auto_loop_active = True - self._update_flow_label() - self._refresh_button_states() - self._schedule_auto_next(first_delay_ms) - return - # Action-driven: the desktop baseline is intentionally lightweight; - # full provider evidence is collected asynchronously after actions. - entry_dump = ( - self._desktop_entry_dump - if self._current_mode() == target.SURFACE_WINDOWS - else None - ) - self._start_recording_session(entry_dump=entry_dump) - - def _set_inspector_topmost(self, enabled: bool) -> None: - try: - self.root.attributes("-topmost", bool(enabled)) - except Exception: - pass - - def _ensure_inspector_visible(self) -> None: - """Keep panel on-screen for progress; do not force-focus (avoids stealing FG).""" - try: - if str(self.root.state()) == "iconic": - self.root.deiconify() - except Exception: - pass - - def _arm_auto_with_countdown(self, *, seconds: int = _START_COUNTDOWN_SEC) -> None: - """Visible countdown, then enter mode-specific auto loop.""" - if not self._flow_is_open(): - return - # Stop prior loop/timers without clearing the countdown we are about to arm. - self._auto_loop_active = False - self._cancel_auto_timer() - self._cancel_countdown() - self._ensure_inspector_visible() - # Stay topmost for progress; capture still uses focus + exclude self. - self._set_inspector_topmost(True) - self._countdown_left = max(1, int(seconds)) - self._update_flow_label() - mode = self._current_mode() - if mode == target.SURFACE_BROWSER: - url = (self.start_url_var.get() or "").strip() - behavior = ( - "Legacy：打开浏览器后每 2 秒采集一次" - if self._is_legacy_interval() - else "打开浏览器后监听用户动作，并采集动作后的稳定状态" - ) - self._set_status( - f"倒计时 {self._countdown_left} 秒：将打开浏览器并进入 {url or '(起始 URL)'}" - ) - self._set_detail( - f"流程：{self.flow_dir}\n\n" - f"- 模式：浏览器自动化\n" - f"- 起始 URL：{url or '(空，自行在浏览器中打开目标站)'}\n" - f"- 倒计时结束后：{behavior}\n" - f"- 可点「结束流程」或取消「自动拾取」中止\n" - ) - else: - behavior = ( - "Legacy：识别窗口后每 2 秒采集一次" - if self._is_legacy_interval() - else "识别窗口后监听用户动作，并采集动作后的稳定状态" - ) - self._set_status( - f"倒计时 {self._countdown_left} 秒：请把要采集的桌面窗口点到最前端" - "（本面板置顶看进度；不强制放大目标窗）。" - ) - self._set_detail( - f"流程：{self.flow_dir}\n\n" - f"- 模式：桌面自动化\n" - f"- {self._countdown_left} 秒内请点击目标窗口（给它焦点）\n" - f"- 倒计时结束后：{behavior}\n" - f"- 可点「结束流程」或取消「自动拾取」中止\n" - ) - self._refresh_button_states() - self._countdown_after_id = self.root.after(1000, self._countdown_tick) - - def _countdown_tick(self) -> None: - self._countdown_after_id = None - if not self._flow_is_open(): - self._countdown_left = 0 - self._refresh_button_states() - return - if not self.auto_pick_var.get(): - self._countdown_left = 0 - self._refresh_button_states() - self._set_status("已取消自动拾取倒计时；可手动点「拾取当前页」。") - return - self._countdown_left -= 1 - if self._countdown_left > 0: - mode = self._current_mode() - if mode == target.SURFACE_BROWSER: - self._set_status( - f"倒计时 {self._countdown_left} 秒：准备打开浏览器…" - ) - else: - self._set_status( - f"倒计时 {self._countdown_left} 秒：请把要采集的窗口点到最前端…" - ) - self._refresh_button_states() - self._countdown_after_id = self.root.after(1000, self._countdown_tick) - return - self._countdown_left = 0 - self._refresh_button_states() - if self._current_mode() == target.SURFACE_BROWSER: - self._set_status("倒计时结束，正在打开浏览器…") - self._begin_auto_browser() - else: - self._set_status("倒计时结束，正在识别最前端窗口…") - self._begin_auto_with_probe() - - def _begin_auto_browser(self) -> None: - """Launch Playwright with start URL, then enter auto loop (no Win32 probe).""" - if self._busy: - return - self._busy = True - self._auto_loop_active = False - self._refresh_button_states() - self._set_inspector_topmost(True) - self._ensure_inspector_visible() - url = (self.start_url_var.get() or "").strip() - status_url = url or "(未填起始 URL，打开空白页后可自行导航)" - self._set_status(f"浏览器模式：正在打开 {status_url} …") - self._set_detail( - f"启动 Playwright…\n\n" - f"- 起始 URL：{url or '(空)'}\n" - f"- profile：{kit_browser.profile_dir()}\n" - f"- 打开完成后进入{'Legacy 间隔采集' if self._is_legacy_interval() else '动作录制'}\n" - ) - - def work() -> None: - try: - # 有 URL 才 force_goto；空则只启浏览器 - sess = kit_browser.ensure_session( - start_url=url or "about:blank", - force_goto=bool(url), - ) - self._post_ui(self._on_browser_ready, sess, url) - except Exception as exc: - self._post_ui(self._on_frontmost_fail, exc) - - threading.Thread(target=work, daemon=True).start() - - def _on_browser_ready(self, sess: dict, url: str) -> None: - self._busy = False - self._refresh_button_states() - live = str(sess.get("url") or "about:blank") - if url and (not live or live == "about:blank" or live.startswith("about:")): - self._on_frontmost_fail( - RuntimeError( - f"ERROR:NAVIGATION_FAILED requested={url!r} actual={live!r}" - ) - ) - return - navigation = dict(sess.get("navigation") or {}) - nav_warning = str(navigation.get("warning") or "") - self._set_status( - f"浏览器已就绪：{live} — " - + ("开始 Legacy 间隔采集。" if self._is_legacy_interval() else "正在采集入口状态。") - ) - self._set_detail( - f"采集目标（Playwright）:\n" - f" mode=browser\n" - f" start_url={url or '(空)'}\n" - f" live_url={live}\n" - f" profile={sess.get('profile_dir')}\n" - f" reused={sess.get('reused')}\n\n" - + (f"导航警告：{nav_warning}\n\n" if nav_warning else "") - + ( - "即将进入 Legacy 自动循环（步间 2 秒）…\n" - if self._is_legacy_interval() - else "即将启动动作录制器…\n" - ) - + ( - "" - if url - else "未填起始 URL：请在弹出的浏览器中打开目标站（空页会自动跳过写入）。\n" - ) - ) - self._set_inspector_topmost(True) - self._ensure_inspector_visible() - self._start_auto_loop(first_delay_ms=_AUTO_START_DELAY_MS) - - def _begin_auto_with_probe(self) -> None: - """Desktop mode: resolve frontmost target, warmup best-effort, then loop.""" - if self._busy: - return - self._busy = True - self._auto_loop_active = False - self._refresh_button_states() - # Keep topmost for progress; target = focused window (Inspector hwnd excluded). - self._set_inspector_topmost(True) - self._ensure_inspector_visible() - self._set_status( - "桌面模式：正在识别最前端窗口（焦点窗=采集目标，已排除本面板）…" - ) - self._set_detail( - "识别最前端窗口中…\n\n" - "- 模式：桌面自动化（强制 UIA）\n" - "- 规则：谁有焦点就采谁（与窗口大小无关）\n" - "- 若是企微：自动调用 kit ensure_ui_ready\n" - ) - exclude_hwnds = self._exclude_hwnds() - - def work() -> None: - try: - uia.ensure_com_sta() - report = uia.prepare_frontmost_for_capture( - exclude_hwnds=exclude_hwnds, - timeout_sec=_FRONTMOST_WAIT_SEC, - ) - self._post_ui(self._on_frontmost_ready, report) - except Exception as exc: - self._post_ui(self._on_frontmost_fail, exc) - - threading.Thread(target=work, daemon=True).start() - - def _on_frontmost_ready(self, report: dict) -> None: - self._busy = False - self._refresh_button_states() - win = report.get("window") or {} - if win.get("pid"): - self.sticky_pid = int(win["pid"]) - bounds = dict(win.get("rect") or win.get("bounds") or {}) - root_row = { - "name": str(win.get("title") or ""), - "value": "", - "automation_id": "", - "control_type": "WindowControl", - "class_name": str(win.get("class_name") or ""), - "framework_id": "Win32", - "is_enabled": True, - "is_offscreen": False, - "bounds": bounds, - "window_scope": dict(win), - } - self._desktop_entry_dump = DumpResult( - surface=target.SURFACE_WINDOWS, - driver="uiautomation-light", - window=dict(win), - tree={**root_row, "children": []}, - index=[dict(root_row)], - locatable=[dict(root_row)], - stats={ - "capture_kind": "entry_light", - "nodes_kept": 1, - "locatable_count": 1, - "provider_enrichment": "deferred", - }, - selection={ - "strategy": str(report.get("strategy") or "foreground_only"), - "entry_light": True, - }, - use_hwnd_screenshot=False, - ) - import kit_wake - - kit = report.get("kit_wake") or {} - kit_line = kit_wake.format_kit_wake_line(kit) - sparse_note = "" - if report.get("provider_cold") or report.get("uia_sparse"): - if kit.get("reason") == "kit_not_installed": - sparse_note = ( - f"（仍偏稀 FindAll={report.get('findall')}：未安装 platform-kit，" - "企微无法走 n2-wake）" - ) - elif kit.get("applicable") and not kit.get("ok"): - sparse_note = ( - f"（kit 暖树未成功 FindAll={report.get('findall')} " - f"code={kit.get('error_code')}；仍按此窗采集）" - ) - else: - sparse_note = ( - f"（UIA 仍偏稀 FindAll={report.get('findall')}；仍按此前端窗采集）" - ) - self._set_status( - f"已锁定桌面窗：{win.get('title')!r} " - f"findall={report.get('findall')} — " - + ("开始 Legacy 间隔采集。" if self._is_legacy_interval() else "正在采集入口状态。") - + sparse_note - ) - self._set_detail( - f"采集目标（桌面 UIA）:\n" - f" mode=windows\n" - f" title={win.get('title')!r}\n" - f" class={win.get('class_name')!r}\n" - f" hwnd={win.get('hwnd')} pid={win.get('pid')}\n" - f" findall={report.get('findall')} sparse={report.get('uia_sparse')} " - f"cold={report.get('provider_cold')}\n" - f" {kit_line}\n\n" - + ( - "即将进入 Legacy 自动循环（步间 2 秒）…\n" - if self._is_legacy_interval() - else "即将启动动作录制器…\n" - ) - ) - self._set_inspector_topmost(True) - self._ensure_inspector_visible() - self._start_auto_loop(first_delay_ms=_AUTO_START_DELAY_MS) - - def _on_frontmost_fail(self, exc: BaseException) -> None: - self._busy = False - self._auto_loop_active = False - # Keep auto checked (default on); user can retry or end flow. - self._set_inspector_topmost(True) - self._ensure_inspector_visible() - self._refresh_button_states() - self._set_status(f"未能锁定最前端目标: {exc}") - messagebox.showerror("UI Inspector", f"未能锁定最前端目标:\n{exc}") - - def _schedule_auto_next(self, delay_ms: int) -> None: - self._cancel_auto_timer() - if not self._auto_loop_active or not self._flow_is_open(): - return - self._auto_after_id = self.root.after(delay_ms, self._auto_loop_tick) - - def _auto_loop_tick(self) -> None: - self._auto_after_id = None - if not self._auto_loop_active or not self._flow_is_open(): - return - if self._busy: - # Should not happen in serial design; re-check soon. - self._schedule_auto_next(200) - return - self.on_pick_page(auto=True) - - def _continue_auto_loop_after_pick(self) -> None: - if self._pending_end: - self._pending_end = False - self._stop_auto_loop() - self._finish_end_flow() - return - if self._auto_loop_active and self._flow_is_open(): - self._set_status( - f"自动循环：第 {self.flow.get('step_count') if self.flow else '?'} 步已处理；" - f"{_AUTO_GAP_MS // 1000} 秒后开始下一拍。点「结束流程」退出循环。" - ) - self._schedule_auto_next(_AUTO_GAP_MS) - - def on_auto_toggle(self) -> None: - if self.auto_pick_var.get(): - if ( - self._flow_is_open() - and not self._auto_loop_active - and not self._busy - and not self._in_start_countdown() - ): - self._arm_auto_with_countdown() - else: - self._update_flow_label() - else: - was = self._auto_loop_active or self._in_start_countdown() - self._stop_auto_loop() - self._update_flow_label() - self._refresh_button_states() - if was and self._flow_is_open(): - self._set_status("已停止自动录制/倒计时；仍可手动点「采集界面检查点」。") - elif self._flow_is_open(): - self._set_status("自动录制已关闭；可手动点「采集界面检查点」。") - - def on_open_out(self) -> None: - path = self.flow_dir if self.flow_dir else self.captures_root - path.mkdir(parents=True, exist_ok=True) - try: - os.startfile(str(path)) # type: ignore[attr-defined] - except Exception: - subprocess.Popen(["explorer", str(path)]) - - def on_start_flow(self) -> None: - if self._busy: - return - mode = self._current_mode() - if mode == target.SURFACE_ANDROID: - messagebox.showwarning( - "UI Inspector", - "手机自动化尚未实现。\n请先选择「浏览器自动化」或「桌面自动化」。", - ) - return - self._persist_prefs() - - if self.flow and self.flow.get("status") == "open": - if not messagebox.askyesno( - "UI Inspector", - "当前已有进行中的流程。\n是否结束旧流程并开始新流程？", - ): - return - self._stop_auto_loop() - self._pending_end = False - try: - art.end_flow(self.flow_dir, self.flow) # type: ignore[arg-type] - except Exception: - pass - self._stop_session() - - name = (self.flow_name_var.get() or "").strip() - if not name: - default_name = _flow_name_for_mode(mode) - name = simpledialog.askstring( - "流程名", "请输入流程名称：", initialvalue=default_name - ) - if not name: - return - self.flow_name_var.set(name) - - try: - self.flow_dir = art.create_flow( - self.captures_root, - flow_name=name, - recording_mode=self._current_recording_mode(), - ) - self.flow = art.load_flow(self.flow_dir) - self._pending_end = False - self._warmed_pids.clear() - self._current_live_action = None - self._desktop_entry_dump = None - self._stop_session() - self._update_flow_label() - self._refresh_button_states() - if self._current_recording_mode() == _MODE_CHECKPOINT: - self.auto_pick_var.set(False) - if self.auto_pick_var.get() and self._current_recording_mode() != _MODE_CHECKPOINT: - # No modal / no minimize: keep panel visible for countdown + progress. - self._arm_auto_with_countdown(seconds=_START_COUNTDOWN_SEC) - else: - self._stop_auto_loop() - self._ensure_inspector_visible() - if mode == target.SURFACE_BROWSER: - url = (self.start_url_var.get() or "").strip() - self._set_status( - f"已开始流程：{self.flow_dir.name}。" - + ( - f"将打开 {url}；请点「采集界面检查点」。" - if url - else "已启浏览器（无起始 URL）；请自行打开目标站后点「采集界面检查点」。" - ) - ) - # Eager open browser so manual pick is ready - def _open() -> None: - try: - kit_browser.ensure_session( - start_url=url or "about:blank", - force_goto=bool(url), - ) - except Exception: - pass - - threading.Thread(target=_open, daemon=True).start() - else: - self._set_status( - f"已开始流程：{self.flow_dir.name}。请翻到目标页后点「采集界面检查点」。" - ) - # Manual mode owns no surface recorder; it only keeps the v2 - # manager available for explicit checkpoints. - self._start_recording_session(checkpoint_only=True) - self._set_detail( - f"流程已创建：{self.flow_dir}\n\n" - f"- 模式：{mode}\n" - f"- 录制：{self._current_recording_mode()}\n" - f"- flow.json / action_flow.json / element_catalog.json / steps/\n" - f"- 手动点「采集界面检查点」；或勾选自动录制用户操作。\n" - ) - except Exception as exc: - messagebox.showerror("UI Inspector", str(exc)) - - def _finish_end_flow(self) -> None: - if not self.flow_dir or not self.flow: - return - if self.flow.get("status") != "open": - self._refresh_button_states() - return - session = self._session - flow_dir = self.flow_dir - flow = self.flow - self._session = None - self._busy = True - self._refresh_button_states() - self._set_status("正在后台刷新动作队列并结束流程……") - - def work() -> None: - try: - if session is not None: - session.stop() - latest_flow = art.load_flow(flow_dir) - ended = art.end_flow(flow_dir, latest_flow) - self._post_ui(self._on_end_flow_finished, ended, None) - except Exception as exc: - self._post_ui(self._on_end_flow_finished, None, exc) - - threading.Thread( - target=work, name="ui-inspector-session-stop", daemon=True - ).start() - - def _on_end_flow_finished( - self, ended_flow: dict | None, error: BaseException | None - ) -> None: - self._busy = False - if error is not None or ended_flow is None: - self._refresh_button_states() - messagebox.showerror("UI Inspector", str(error or "结束流程失败")) - return - try: - self.flow = ended_flow - self._update_flow_label() - self._refresh_button_states() - n = self.flow.get("step_count") or 0 - v2 = self.flow.get("v2") or {} - self._set_status( - f"流程已结束：actions={v2.get('action_count', '?')}，" - f"states={v2.get('state_count', '?')}，" - f"observations={v2.get('observation_count', '?')}，" - f"legacy_steps={n}。" - "可再点「开始流程」开新文档。" - ) - self._set_detail( - f"流程已结束：{self.flow_dir}\n\n" - f"- v1 steps: {n}\n" - f"- v2 actions: {v2.get('action_count')}\n" - f"- 请打开 SUMMARY.md / action_flow.json / element_catalog.json\n" - ) - messagebox.showinfo( - "UI Inspector", - f"流程已结束\nactions={v2.get('action_count')} " - f"states={v2.get('state_count')} " - f"observations={v2.get('observation_count')} " - f"legacy_steps={n}\n{self.flow_dir}", - ) - except Exception as exc: - messagebox.showerror("UI Inspector", str(exc)) - - def on_pause_resume(self) -> None: - if not self._session or not self._flow_is_open(): - return - if self._rec_paused: - self._session.resume() - self._rec_paused = False - self.pause_btn.configure(text="暂停") - self._set_status("已继续录制。") - else: - self._session.pause() - self._rec_paused = True - self.pause_btn.configure(text="继续") - self._set_status("已暂停录制（Inspector 自身操作不会写入）。") - self._update_rec_stats() - - def on_end_flow(self) -> None: - if not self.flow_dir or not self.flow: - messagebox.showwarning("UI Inspector", "没有进行中的流程。") - return - if self.flow.get("status") != "open": - messagebox.showinfo("UI Inspector", "流程已经结束。") - return - - self._stop_auto_loop() - # Keep「自动录制」勾选状态（默认开），方便下一轮直接开始。 - - if self._busy: - self._pending_end = True - self._set_status("已请求结束：等待当前这一拍 dump/保存完成…") - self._refresh_button_states() - return - - self._pending_end = False - self._finish_end_flow() - - def on_pick_page(self, *, auto: bool = False) -> None: - """Manual checkpoint or legacy auto dump. Action-driven mode uses Recorder.""" - if self._busy: - return - if not self._flow_is_open(): - if not auto: - messagebox.showwarning("UI Inspector", "请先点「开始流程」。") - return - if auto and not self._auto_loop_active: - return - # In action-driven mode, auto ticks should not dump steps. - if auto and not self._is_legacy_interval(): - return - - self._busy = True - self._refresh_button_states() - self._show_picking_placeholder(auto=auto) - self.root.update_idletasks() - - # Capture Tk-owned configuration before entering the worker. - mode = self._current_mode() - under_cursor = bool(self.under_cursor_var.get()) - prefer_search = bool(self.prefer_wecom_search_var.get()) - start_url = (self.start_url_var.get() or "").strip() - sticky_pid = self.sticky_pid - exclude_hwnds = self._exclude_hwnds() - flow_dir = self.flow_dir - synthetic_browser_info = ( - self._synthetic_browser_info() - if mode == target.SURFACE_BROWSER - else None - ) - - def work() -> None: - try: - uia.ensure_com_sta() - flow = art.load_flow(flow_dir) # type: ignore[arg-type] - if flow.get("status") != "open": - raise RuntimeError("流程已结束，取消本步拾取。") - if mode == target.SURFACE_ANDROID: - raise RuntimeError( - "ERROR:SURFACE_UNSUPPORTED 手机自动化尚未实现。" - ) - surface = mode - sel_meta: dict = { - "pick_mode": mode, - "sticky_pid": sticky_pid, - } - if surface == target.SURFACE_BROWSER: - kit_browser.ensure_session( - start_url=start_url or "about:blank", - force_goto=False, - ) - info = synthetic_browser_info - # Best-effort Win32 chrome window for meta only - try: - win_info, cands, win_meta = uia.resolve_capture_target( - exclude_hwnds=exclude_hwnds, - prefer_pid=None, - under_cursor=False, - prefer_wecom_search_popup=False, - ) - if target.surface_from_window_info(win_info) == ( - target.SURFACE_BROWSER - ): - info = win_info - sel_meta.update(win_meta) - sel_meta["candidate_count"] = len(cands) - except Exception: - sel_meta["strategy"] = "playwright_session" - sel_meta["resolved_surface"] = surface - sel_meta["start_url"] = start_url - else: - info, cands, win_meta = uia.resolve_capture_target( - exclude_hwnds=exclude_hwnds, - prefer_pid=None if auto else sticky_pid, - under_cursor=under_cursor, - prefer_wecom_search_popup=prefer_search, - ) - sel_meta.update(win_meta) - sel_meta["candidate_count"] = len(cands) - sel_meta["resolved_surface"] = surface - driver = get_driver(surface) - manual_warm_key = ( - (int(info.pid), str(info.class_name or "")) - if surface == target.SURFACE_WINDOWS - else int(info.pid or 0) - ) - warmup_mode = ( - "light" if manual_warm_key in self._warmed_pids else "full" - ) - result = driver.dump( - info=info, - selection_meta=sel_meta, - max_depth=self.max_depth, - max_nodes=self.max_nodes, - warmup_mode=warmup_mode, - warmed_pids=self._warmed_pids, - ) - if info.pid: - self._warmed_pids.add(manual_warm_key) - skip_reason = skip_policy.should_skip_empty_dump( - surface=result.surface, - url=result.url, - locatable=result.locatable, - stats=result.stats, - ) - if skip_reason: - msg = skip_policy.skip_reason_text( - skip_reason, url=result.url - ) - self._post_ui( - self._on_pick_skipped, - msg, - auto=auto, - surface=result.surface, - url=result.url, - ) - return - # Dual-write v2 checkpoint when session exists (manual) - if self._session is not None and not auto: - try: - mgr = self._session.manager - before = mgr.snapshots.latest - after = mgr.snapshots.capture_from_dump( - result, change_reason="checkpoint", force_persist=True - ) - mgr._finalize_action( - action_type="checkpoint", - before=before, - after=after, - target={}, - input_payload={}, - raw_event={"meta": {"source": "manual_checkpoint"}}, - confidence=0.9, - ) - except Exception: - pass - window = result.window - locatable = result.locatable - index = result.index - stats = result.stats - label = art.resolve_step_label( - flow, - window_title=window.get("title") or info.title or "", - window_class=window.get("class_name") or info.class_name or "", - locatable=locatable, - ) - step_dir, step_idx, step_name = art.add_step_dir( - self.flow_dir, # type: ignore[arg-type] - flow, - window_title=window.get("title") or info.title or "", - window_class=window.get("class_name") or info.class_name or "", - locatable=locatable, - label=label, - ) - meta = art.build_meta( - window=window, - scene=flow.get("flow_name") or "flow", - surface=result.surface, - driver=result.driver, - url=result.url, - extra={ - "flow_dir": str(self.flow_dir), - "step_index": step_idx, - "step_name": step_name, - "step_label": label, - "auto_pick": auto, - "dump_stats": stats, - "selection": result.selection, - }, - ) - art.write_json(step_dir / "meta.json", meta) - art.write_json(step_dir / "tree.json", result.tree) - art.write_jsonl(step_dir / "index.jsonl", index) - art.write_json(step_dir / "locatable.json", locatable) - png_ok = False - if result.screenshot_bytes: - try: - (step_dir / "screen.png").write_bytes(result.screenshot_bytes) - png_ok = True - except Exception: - png_ok = False - elif result.use_hwnd_screenshot: - png_ok = art.capture_window_png(info.hwnd, step_dir / "screen.png") - noteworthy = art.noteworthy_from_index(index) - art.write_step_summary( - step_dir, meta=meta, stats=stats, noteworthy=noteworthy - ) - art.register_step( - self.flow_dir, # type: ignore[arg-type] - flow, - step_index=step_idx, - step_name=step_name, - window=window, - stats=stats, - label=label, - surface=result.surface, - url=result.url, - ) - flow = art.load_flow(self.flow_dir) # type: ignore[arg-type] - - import kit_wake - - warm = stats.get("warmup") or {} - kit = stats.get("kit_wake") or {} - kit_line = kit_wake.format_kit_wake_line(kit) - sparse_note = "" - if stats.get("uia_sparse"): - sparse_note = "\n⚠ sparse=true：可定位候选偏少，请结合 screen.png。\n" - mode = "自动" if auto else "手动" - url_line = f"url: {result.url}\n" if result.url else "" - header = ( - f"流程步骤: #{step_idx} `{step_name}`（{mode}）\n" - f"surface: {result.surface} ({target.surface_label(result.surface)}) " - f"driver={result.driver}\n" - f"{url_line}" - f"窗口: title={window.get('title')!r} class={window.get('class_name')!r} " - f"pid={window.get('pid')} hwnd={window.get('hwnd')}\n" - f"模式: {mode} 选窗: {sel_meta.get('strategy')}\n" - f"kit: {kit_line}\n" - f"软poke: findall {warm.get('findall_before')} → {warm.get('findall_after')} " - f"woke={warm.get('woke')}\n" - f"统计: kept={stats.get('nodes_kept')} findall={stats.get('findall_count')} " - f"locatable={stats.get('locatable_count')} png={png_ok} " - f"sparse={stats.get('uia_sparse')} cold={stats.get('provider_cold')} " - f"merge={stats.get('merge_same_pid_allowed')}\n" - f"已写入: {step_dir}\n" - f"流程文档: {self.flow_dir}\n" - f"{sparse_note}" - f"{'=' * 60}\n" - ) - body = ( - header - + "\n" - + (result.locatable_text or "") - + "\n\n" - + ("=" * 60) - + "\n控件树:\n" - + (result.tree_text or "") - ) - self._post_ui( - self._on_pick_ok, - window, - flow, - body, - step_name, - auto=auto, - surface=result.surface, - ) - except Exception as exc: - self._post_ui(self._on_fail, exc, auto=auto) - - threading.Thread(target=work, daemon=True).start() - - def _on_pick_ok( - self, - window: dict, - flow: dict, - text: str, - step_name: str, - *, - auto: bool = False, - surface: str = "windows", - ) -> None: - self._busy = False - self.flow = flow - self.last_window = dict(window or {}) - if window.get("pid"): - try: - self.sticky_pid = int(window["pid"]) - except Exception: - pass - self._update_flow_label() - self._refresh_button_states() - url = window.get("url") or "" - url_bit = f" url={url}" if url else "" - self.target_var.set( - f"本步: surface={surface} title={window.get('title')!r} " - f"class={window.get('class_name')!r}{url_bit} | step=`{step_name}`" - ) - self._set_detail(text) - if not auto: - self._set_status( - f"第 {flow.get('step_count')} 步已写入。可换页继续拾取，或点「结束流程」。" - ) - try: - self.root.lift() - self.root.attributes("-topmost", True) - except Exception: - pass - if self._pending_end: - self._pending_end = False - self._finish_end_flow() - return - - # auto: either pending end, or continue loop after gap - self._continue_auto_loop_after_pick() - - def _on_pick_skipped( - self, - message: str, - *, - auto: bool = False, - surface: str = "browser", - url: str = "", - ) -> None: - """Blank/empty browser dump: do not write a step; keep looping if auto.""" - self._busy = False - self._refresh_button_states() - self._update_flow_label() - url_bit = f" url={url}" if url else "" - self.target_var.set(f"已跳过空页（{surface}）{url_bit}") - n = int((self.flow or {}).get("step_count") or 0) - detail = ( - f"{message}\n\n" - f"- 未写入 steps/（避免 about:blank / 空壳污染流程文档）\n" - f"- 当前流程已有步骤数: {n}\n" - f"- 打开目标站或等待页面加载后再拾取\n" - ) - if auto: - detail += f"- 自动循环将在 {_AUTO_GAP_MS // 1000} 秒后继续\n" - self._set_detail(detail) - self._set_status(message) - if not auto: - try: - self.root.lift() - self.root.attributes("-topmost", True) - except Exception: - pass - if self._pending_end: - self._pending_end = False - self._finish_end_flow() - return - self._continue_auto_loop_after_pick() - - def _on_fail(self, exc: BaseException, *, auto: bool = False) -> None: - self._busy = False - self._refresh_button_states() - self._set_status(f"失败: {exc}") - if not auto: - messagebox.showerror("UI Inspector", str(exc)) - if self._pending_end: - self._pending_end = False - self._finish_end_flow() - return - # Auto: do not pop modal; keep looping after gap (unless ending) - self._continue_auto_loop_after_pick() - - def _on_close(self) -> None: - if self._closed: - return - self._closed = True - self._stop_auto_loop() - self._pending_end = False - self._stop_session() - if self.flow_dir and self.flow and self.flow.get("status") == "open": - try: - latest_flow = art.load_flow(self.flow_dir) - self.flow = art.end_flow(self.flow_dir, latest_flow) - except Exception: - pass - try: - self._persist_prefs() - except Exception: - pass - try: - kit_browser.shutdown() - except Exception: - pass - try: - self.root.destroy() - except Exception: - pass - - def run(self) -> None: - self.captures_root.mkdir(parents=True, exist_ok=True) - try: - self.root.mainloop() - except KeyboardInterrupt: - pass - finally: - self._on_close() - - -def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - p = argparse.ArgumentParser( - description=( - f"OpenClaw UI Inspector v{__version__}" - "（browser/windows：动作录制、检查点与 Legacy 间隔采集）" - ) - ) - p.add_argument("--out", type=Path, default=None, help="输出根目录") - p.add_argument( - "--scene", - default="", - help="默认流程名（空则按拾取模式：browser / desktop / mobile）", - ) - p.add_argument("--max-depth", type=int, default=16) - p.add_argument("--max-nodes", type=int, default=8000) - return p.parse_args(argv) - - -def main(argv: list[str] | None = None) -> int: - args = parse_args(argv) - out = args.out.resolve() if args.out else art.default_captures_root(_HERE) - try: - uia.ensure_uia() - except Exception as exc: - print(f"ERROR: {exc}", file=sys.stderr) - return 1 - app = InspectorApp( - captures_root=out, - scene=args.scene, - max_depth=args.max_depth, - max_nodes=args.max_nodes, - ) - app.run() - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/ui_inspector/artifacts.py b/tools/ui_inspector/artifacts.py deleted file mode 100644 index 45ad7b5..0000000 --- a/tools/ui_inspector/artifacts.py +++ /dev/null @@ -1,735 +0,0 @@ -"""Flow-oriented capture artifacts: one flow document, many steps.""" - -from __future__ import annotations - -import json -import re -from datetime import datetime -from pathlib import Path -from typing import Any, Optional - -try: - from . import __version__ -except ImportError: - try: - import importlib.util - from pathlib import Path - - _spec = importlib.util.spec_from_file_location( - "ui_inspector_pkg", Path(__file__).resolve().parent / "__init__.py" - ) - _mod = importlib.util.module_from_spec(_spec) # type: ignore[arg-type] - assert _spec and _spec.loader - _spec.loader.exec_module(_mod) - __version__ = getattr(_mod, "__version__", "0.16.0") - except Exception: - __version__ = "0.16.0" - - -def _local_ts() -> str: - """Local wall-clock stamp for capture folder names: YYYYMMDDHHMMSS.""" - return datetime.now().astimezone().strftime("%Y%m%d%H%M%S") - - -def safe_slug(text: str, *, fallback: str = "session") -> str: - text = (text or "").strip() - if not text: - return fallback - text = re.sub(r"\s+", "-", text) - text = re.sub(r"[^\w\u4e00-\u9fff\-]+", "", text, flags=re.UNICODE) - text = text.strip("-_") or fallback - return text[:80] - - -def _clean_ui_text(text: str) -> str: - """Strip WeCom-style rich markup and collapse whitespace.""" - s = (text or "").strip() - if not s: - return "" - s = re.sub(r"<[^>]+>", "", s) - s = re.sub(r"\s+", " ", s).strip() - return s - - -# Name/AutomationId keys that usually mark "what page am I on" -_PAGE_HINT_KEYS: dict[str, int] = { - "panel_title_label": 100, - "title": 95, - "ww_btn_text": 70, - "main_chattext": 45, -} - -_LAYOUT_NOISE = { - "bg", - "navigation_bar", - "outer_layout", - "content_container", - "title_layout", - "closebtn", - "VerticalLayoutUI", - "HorizontalLayoutUI", - "wbtn_container", - "search_layout", - "search_layout_border", - "inner_search_layout", - "ControlUI", - "LabelUI", - "item", - "externals", - "status_layout", - "search_button_region", - "search_icon", - "ww_btn_icon", - "wbadge_label", - "lasttime", - "groupbuddydescription", - "bottom_text", - "chattext", - "pane", - "Pane", -} - -_CHROME_TEXT = { - "关闭", - "最小化", - "最大化", - "还原", - "取消", - "确定", - "关闭窗口", -} - -# Left-rail / chrome values that appear on every WeCom page — not page identity -_GLOBAL_NAV_TEXT = { - "消息", - "邮件", - "文档", - "日程", - "待办", - "会议", - "智能文档", - "通讯录", - "微盘", - "高级功能", - "工作台", - "收藏", - "智能总结", - "回到最新位置", -} - - -def _row_key(r: dict[str, Any]) -> str: - return _clean_ui_text(str(r.get("name") or r.get("automation_id") or "")) - - -def _looks_like_search_dialog( - window_title: str, window_class: str, rows: list[dict[str, Any]] -) -> bool: - t = window_title or "" - c = window_class or "" - if "Search" in c or "搜索" in t or "添加客户" in t or "添加联系人" in t: - return True - for r in rows: - if _row_key(r) == "title": - v = _clean_ui_text(str(r.get("value") or "")) - if "添加" in v or "搜索" in v: - return True - return False - - -def _short_entity(text: str, *, limit: int = 16) -> str: - """Clean contact/chat title: strip markup leftovers and @微信 suffix.""" - s = _clean_ui_text(text) - if not s: - return "" - s = re.sub(r"@微信\s*$", "", s) - s = re.sub(r"@WeCom\s*$", "", s, flags=re.I) - s = s.strip(" -_|") - if len(s) > limit: - s = s[:limit].rstrip("-_") - return s - - -def _raw_value(r: dict[str, Any]) -> str: - return str(r.get("value") or "") - - -def _is_selected_buddy(raw_value: str) -> bool: - # WeCom marks the active conversation with semi-transparent white color markup. - return "#99ffffff" in (raw_value or "").lower() - - -def derive_step_label( - *, - window_title: str = "", - window_class: str = "", - locatable: Optional[list[dict[str, Any]]] = None, - max_hints: int = 2, -) -> str: - """ - Build a human-readable step label from window + page-specific UIA hints. - - Prefers: search box text, page title, selected chat, compose draft, action buttons. - Avoids bare「会话列表」when a concrete chat/search/input signal exists. - """ - rows = list(locatable or []) - base = _clean_ui_text(window_title) - if not base: - for r in rows: - if (r.get("control_type") or "") == "WindowControl": - base = _clean_ui_text(str(r.get("name") or "")) - if base: - break - if not base and window_class: - base = window_class - - has_panel_title = any( - _row_key(r) == "panel_title_label" and _clean_ui_text(str(r.get("value") or "")) - for r in rows - ) - has_groupbuddy = any(_row_key(r) == "groupbuddyname" for r in rows) - has_search_result = any(_row_key(r) == "search_result" for r in rows) - has_searchedit = any(_row_key(r) in {"searchedit", "search_edit"} for r in rows) - search_dialog = _looks_like_search_dialog(window_title, window_class, rows) - - scored: list[tuple[int, str]] = [] - searchedit_value = "" - selected_buddy = "" - first_buddy = "" - - for r in rows: - key = _row_key(r) - if not key or key in _LAYOUT_NOISE: - continue - raw = _raw_value(r) - value = _clean_ui_text(raw) - name_disp = _clean_ui_text(str(r.get("name") or "")) - ctype = str(r.get("control_type") or "") - - if key in {"searchedit", "search_edit"} and value: - searchedit_value = _short_entity(value, limit=20) - # Main-window search AND add-friend dialog both matter. - scored.append((98, f"搜索{searchedit_value}")) - continue - - if key == "send_rich_edit" and value: - draft = _short_entity(value, limit=12) - if draft: - scored.append((86, f"输入{draft}")) - continue - - if key == "groupbuddyname" and value: - entity = _short_entity(value, limit=14) - if not entity: - continue - if _is_selected_buddy(raw) and not selected_buddy: - selected_buddy = entity - scored.append((92, entity)) - elif not first_buddy and not r.get("is_offscreen"): - first_buddy = entity - continue - - pri = _PAGE_HINT_KEYS.get(key) - if pri is not None and value: - if value in _CHROME_TEXT or value in _GLOBAL_NAV_TEXT: - continue - if key == "title" and value == base: - continue - if key == "main_chattext" and (has_panel_title or has_groupbuddy): - continue - entity = _short_entity(value, limit=16) - if not entity: - continue - scored.append((pri, entity)) - continue - - if ctype in { - "ButtonControl", - "EditControl", - "TabItemControl", - "TextControl", - "HyperlinkControl", - "MenuItemControl", - "CheckBoxControl", - "RadioButtonControl", - "ComboBoxControl", - "ListItemControl", - }: - label = _short_entity(value or name_disp, limit=16) - if ( - not label - or label in _CHROME_TEXT - or label in _LAYOUT_NOISE - or label in _GLOBAL_NAV_TEXT - ): - continue - scored.append((40, label)) - - # Conversation context when no selected buddy / search text yet - if ( - has_groupbuddy - and not selected_buddy - and first_buddy - and not has_panel_title - and not searchedit_value - ): - scored.append((78, first_buddy)) - - strong = any(p >= 90 for p, _ in scored) - if search_dialog and has_searchedit and not searchedit_value and not strong: - scored.append((75, "搜索框")) - elif search_dialog and has_search_result and searchedit_value and not strong: - scored.append((50, "搜索结果")) - elif ( - has_groupbuddy - and not has_panel_title - and not strong - and not first_buddy - and not selected_buddy - ): - scored.append((60, "会话列表")) - - scored.sort(key=lambda x: (-x[0], x[1])) - hints: list[str] = [] - seen: set[str] = set() - for _pri, text in scored: - if not text or text == base or text in seen: - continue - if text in _CHROME_TEXT or text in _GLOBAL_NAV_TEXT: - continue - compact = ( - text[2:] if text.startswith(("搜索", "输入")) and len(text) > 2 else text - ) - if compact and any( - compact == s or s.endswith(compact) or compact in s for s in seen - ): - continue - seen.add(text) - if compact: - seen.add(compact) - hints.append(text) - if len(hints) >= max_hints: - break - - parts = [p for p in [base, *hints] if p] - if not parts: - parts = ["step"] - return "_".join(parts) - - -def _label_slug_key(label: str) -> str: - return safe_slug(label, fallback="step") - - -def _unique_step_dirname(steps_root: Path, idx: int, label: str) -> str: - """NN_label, with -2/-3 suffix if the folder already exists.""" - slug = _label_slug_key(label) - base = f"{idx:02d}_{slug}" - if not (steps_root / base).exists(): - return base - n = 2 - while (steps_root / f"{base}-{n}").exists(): - n += 1 - return f"{base}-{n}" - - -def _existing_label_slugs(flow: dict[str, Any]) -> set[str]: - out: set[str] = set() - for s in flow.get("steps") or []: - lab = s.get("label") or "" - name = str(s.get("name") or "") - if lab: - out.add(_label_slug_key(str(lab))) - m = re.match(r"^\d{2}_(.+?)(?:-\d+)?$", name) - if m: - out.add(m.group(1)) - return out - - -def resolve_step_label( - flow: dict[str, Any], - *, - window_title: str = "", - window_class: str = "", - locatable: Optional[list[dict[str, Any]]] = None, -) -> str: - """Derive label; if it collides with earlier steps, add more hints.""" - used = _existing_label_slugs(flow) - label = derive_step_label( - window_title=window_title, - window_class=window_class, - locatable=locatable, - max_hints=2, - ) - if _label_slug_key(label) not in used: - return label - label3 = derive_step_label( - window_title=window_title, - window_class=window_class, - locatable=locatable, - max_hints=3, - ) - if _label_slug_key(label3) not in used: - return label3 - # Last resort: keep richest label; folder uniquifier adds -2 - return label3 - - -def default_captures_root(tool_dir: Optional[Path] = None) -> Path: - base = tool_dir or Path(__file__).resolve().parent - return base / "_captures" - - -def write_json(path: Path, data: Any) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text( - json.dumps(data, ensure_ascii=False, indent=2) + "\n", - encoding="utf-8", - ) - - -def write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("w", encoding="utf-8") as fh: - for row in rows: - fh.write(json.dumps(row, ensure_ascii=False) + "\n") - - -def read_json(path: Path) -> dict[str, Any]: - return json.loads(path.read_text(encoding="utf-8")) - - -def build_meta( - *, - window: dict[str, Any], - scene: str, - skill_root: Optional[str] = None, - extra: Optional[dict[str, Any]] = None, - surface: str = "windows", - driver: str = "uiautomation", - url: str = "", -) -> dict[str, Any]: - now = datetime.now().astimezone() - meta = { - "schema": "openclaw.ui_inspector.capture.v1", - "surface": surface or "windows", - "captured_at": now.isoformat(), - "scene": scene, - "skill_root": skill_root, - "app": { - "title": window.get("title"), - "class_name": window.get("class_name"), - "process_name": window.get("process_name"), - "pid": window.get("pid"), - "hwnd": window.get("hwnd"), - "url": url or window.get("url") or "", - }, - "environment": { - "os": "windows", - "tz": now.tzname(), - }, - "tool": { - "name": "openclaw-ui-inspector", - "version": __version__, - "driver": driver or "uiautomation", - }, - } - if extra: - meta["extra"] = extra - return meta - - -def noteworthy_from_index(index: list[dict[str, Any]], *, limit: int = 80) -> list[dict[str, Any]]: - rows = [ - r - for r in index - if (r.get("name") or r.get("value") or r.get("automation_id")) - ] - rows.sort( - key=lambda r: ( - 0 if r.get("value") else 1, - 0 if r.get("name") else 1, - int(r.get("depth") or 0) if isinstance(r.get("depth"), int) and r.get("depth") >= 0 else 99, - ) - ) - return rows[:limit] - - -def capture_window_png(hwnd: int, dest: Path) -> bool: - try: - from PIL import ImageGrab - except ImportError: - return False - try: - import ctypes - from ctypes import wintypes - - user32 = ctypes.windll.user32 - rect = wintypes.RECT() - if not user32.GetWindowRect(int(hwnd), ctypes.byref(rect)): - return False - bbox = (int(rect.left), int(rect.top), int(rect.right), int(rect.bottom)) - if bbox[2] <= bbox[0] or bbox[3] <= bbox[1]: - return False - img = ImageGrab.grab(bbox=bbox, all_screens=True) - dest.parent.mkdir(parents=True, exist_ok=True) - img.save(str(dest)) - return True - except Exception: - try: - img = ImageGrab.grab(all_screens=True) - dest.parent.mkdir(parents=True, exist_ok=True) - img.save(str(dest)) - return True - except Exception: - return False - - -# --- Flow document API ------------------------------------------------- - -def create_flow( - captures_root: Path, - *, - flow_name: str = "desktop", - recording_mode: str = "auto", -) -> Path: - """开始流程：新建一份流程文档目录（v1 steps + 可选 v2 资产目录）。""" - captures_root.mkdir(parents=True, exist_ok=True) - slug = safe_slug(flow_name, fallback="flow") - flow_dir = captures_root / f"{_local_ts()}_{slug}" - flow_dir.mkdir(parents=True, exist_ok=False) - (flow_dir / "steps").mkdir(parents=True, exist_ok=True) - # v2 dual-write directories (empty until recorder writes) - for sub in ("states", "observations", "actions", "elements", "journal", "diagnostics"): - (flow_dir / sub).mkdir(parents=True, exist_ok=True) - now = datetime.now().astimezone().isoformat() - flow = { - "schema": "openclaw.ui_inspector.flow.v1", - "flow_name": flow_name, - "status": "open", - "created_at": now, - "updated_at": now, - "ended_at": None, - "step_count": 0, - "steps": [], - "tool": { - "name": "openclaw-ui-inspector", - "version": __version__, - }, - # Optional v2 pointers — old readers ignore unknown fields. - "v2": { - "recording_mode": recording_mode or "auto", - "action_flow": "action_flow.json", - "element_catalog": "element_catalog.json", - "states_dir": "states/", - "observations_dir": "observations/", - "actions_dir": "actions/", - "elements_dir": "elements/", - "raw_event_journal": "journal/raw_events.jsonl", - "pages": "pages.json", - "diagnostics_dir": "diagnostics/", - "validation": "diagnostics/validation.json", - "replay_report": "diagnostics/replay_report.json", - }, - } - write_json(flow_dir / "flow.json", flow) - write_flow_summary(flow_dir, flow) - return flow_dir - - -def load_flow(flow_dir: Path) -> dict[str, Any]: - return read_json(flow_dir / "flow.json") - - -def save_flow(flow_dir: Path, flow: dict[str, Any]) -> None: - flow["updated_at"] = datetime.now().astimezone().isoformat() - write_json(flow_dir / "flow.json", flow) - write_flow_summary(flow_dir, flow) - - -def next_step_index(flow: dict[str, Any]) -> int: - return int(flow.get("step_count") or 0) + 1 - - -def add_step_dir( - flow_dir: Path, - flow: dict[str, Any], - *, - window_title: str = "", - window_class: str = "", - locatable: Optional[list[dict[str, Any]]] = None, - label: Optional[str] = None, -) -> tuple[Path, int, str]: - """Allocate steps/NN_/ under an open flow.""" - if flow.get("status") != "open": - raise RuntimeError("当前流程已结束，请先「开始流程」。") - idx = next_step_index(flow) - if not label: - label = resolve_step_label( - flow, - window_title=window_title, - window_class=window_class, - locatable=locatable, - ) - steps_root = flow_dir / "steps" - steps_root.mkdir(parents=True, exist_ok=True) - step_name = _unique_step_dirname(steps_root, idx, label) - step_dir = steps_root / step_name - step_dir.mkdir(parents=True, exist_ok=False) - return step_dir, idx, step_name - - -def register_step( - flow_dir: Path, - flow: dict[str, Any], - *, - step_index: int, - step_name: str, - window: dict[str, Any], - stats: dict[str, Any], - label: Optional[str] = None, - surface: str = "windows", - url: str = "", -) -> dict[str, Any]: - entry = { - "index": step_index, - "name": step_name, - "label": label or step_name, - "captured_at": datetime.now().astimezone().isoformat(), - "surface": surface or "windows", - "url": url or window.get("url") or "", - "window_title": window.get("title"), - "window_class": window.get("class_name"), - "pid": window.get("pid"), - "hwnd": window.get("hwnd"), - "findall": stats.get("findall_count"), - "locatable": stats.get("locatable_count"), - "kept": stats.get("nodes_kept"), - "uia_sparse": stats.get("uia_sparse"), - "path": f"steps/{step_name}", - } - steps = list(flow.get("steps") or []) - steps.append(entry) - flow["steps"] = steps - flow["step_count"] = len(steps) - save_flow(flow_dir, flow) - return entry - - -def end_flow(flow_dir: Path, flow: dict[str, Any]) -> dict[str, Any]: - if flow.get("status") == "ended": - return flow - flow["status"] = "ended" - flow["ended_at"] = datetime.now().astimezone().isoformat() - save_flow(flow_dir, flow) - return flow - - -def write_flow_summary(flow_dir: Path, flow: dict[str, Any]) -> None: - lines = [ - f"# 流程：{flow.get('flow_name')}", - "", - f"- status: `{flow.get('status')}`", - f"- created_at: `{flow.get('created_at')}`", - f"- updated_at: `{flow.get('updated_at')}`", - f"- ended_at: `{flow.get('ended_at')}`", - f"- step_count: `{flow.get('step_count')}`", - "", - "## 步骤", - "", - ] - steps = flow.get("steps") or [] - if not steps: - lines.append("（尚无拾取步骤。点「拾取当前页」追加。）") - lines.append("") - else: - for s in steps: - lines.append(f"### {s.get('index')}. `{s.get('name')}`") - lines.append("") - if s.get("label"): - lines.append(f"- label: `{s.get('label')}`") - lines.append(f"- surface: `{s.get('surface') or 'windows'}`") - if s.get("url"): - lines.append(f"- url: `{s.get('url')}`") - lines.append( - f"- window: `{s.get('window_title')}` / `{s.get('window_class')}`" - ) - lines.append( - f"- stats: findall=`{s.get('findall')}` locatable=`{s.get('locatable')}` " - f"kept=`{s.get('kept')}` sparse=`{s.get('uia_sparse')}`" - ) - lines.append(f"- path: `{s.get('path')}/`") - lines.append("") - lines.extend( - [ - "## 说明", - "", - "- 一个流程对应本目录一份文档（`flow.json` + `SUMMARY.md` + `steps/`）。", - "- 每拾取一页，在 `steps/` 下新增一步；给 AI 时优先读本 `SUMMARY.md` 与各步 `locatable.json`。", - "- `surface=windows` 为桌面 UIA；`surface=browser` 为 Playwright 整页 dump。", - "- Name ≠ 界面中文时请看 Value；浏览器步优先看 `primary_locator` / `playwright_candidates`。", - "- v2（若存在）：`action_flow.json` + `element_catalog.json` + `states/` + `actions/` + `elements/`。", - "- 给 AI 编程工具时优先提供 `action_flow.json` 与 `element_catalog.json`，诊断再用 steps/ 原始树。", - "", - ] - ) - (flow_dir / "SUMMARY.md").write_text("\n".join(lines), encoding="utf-8") - - -def write_step_summary( - step_dir: Path, - *, - meta: dict[str, Any], - stats: Optional[dict[str, Any]] = None, - noteworthy: Optional[list[dict[str, Any]]] = None, -) -> None: - app = meta.get("app") or {} - tool = meta.get("tool") or {} - lines = [ - "# Step Summary", - "", - f"- captured_at: `{meta.get('captured_at')}`", - f"- surface: `{meta.get('surface')}` driver: `{tool.get('driver')}`", - f"- window: `{app.get('title')}` / `{app.get('class_name')}` pid=`{app.get('pid')}`", - "", - ] - if app.get("url"): - lines.append(f"- url: `{app.get('url')}`") - lines.append("") - if stats: - lines.extend( - [ - "## Stats", - "", - f"- kept: `{stats.get('nodes_kept')}`", - f"- findall: `{stats.get('findall_count')}`", - f"- locatable: `{stats.get('locatable_count')}`", - f"- uia_sparse: `{stats.get('uia_sparse')}`", - "", - ] - ) - if noteworthy: - lines.extend(["## Notable", ""]) - for item in noteworthy[:40]: - primary = (item.get("primary_locator") or {}).get("expr") or "" - extra = f" locator=`{primary}`" if primary else "" - lines.append( - f"- `{item.get('control_type')}` Name=`{item.get('name')}` " - f"Value=`{item.get('value') or item.get('text')}`{extra}" - ) - lines.append("") - lines.extend( - [ - "## Files", - "", - "- `meta.json` / `tree.json` / `index.jsonl` / `locatable.json` / `screen.png`", - "", - ] - ) - (step_dir / "SUMMARY.md").write_text("\n".join(lines), encoding="utf-8") - - - -# Back-compat aliases used by older smoke scripts -def create_session_dir(captures_root: Path, *, scene: str = "desktop") -> Path: - return create_flow(captures_root, flow_name=scene) diff --git a/tools/ui_inspector/compatibility/__init__.py b/tools/ui_inspector/compatibility/__init__.py deleted file mode 100644 index 49ec57b..0000000 --- a/tools/ui_inspector/compatibility/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""Backward-compatible readers and entry points.""" - diff --git a/tools/ui_inspector/compatibility/v1_reader.py b/tools/ui_inspector/compatibility/v1_reader.py deleted file mode 100644 index fbd821b..0000000 --- a/tools/ui_inspector/compatibility/v1_reader.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Legacy flow.json reader.""" - -from artifacts import load_flow - -__all__ = ["load_flow"] - diff --git a/tools/ui_inspector/compatibility/v1_writer.py b/tools/ui_inspector/compatibility/v1_writer.py deleted file mode 100644 index f46ca1e..0000000 --- a/tools/ui_inspector/compatibility/v1_writer.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Legacy steps/ writer exports.""" - -from artifacts import add_step_dir, create_flow, save_flow - -__all__ = ["add_step_dir", "create_flow", "save_flow"] - diff --git a/tools/ui_inspector/core/__init__.py b/tools/ui_inspector/core/__init__.py deleted file mode 100644 index ad8b7f7..0000000 --- a/tools/ui_inspector/core/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""Surface-independent UI Inspector domain and recording pipeline.""" - diff --git a/tools/ui_inspector/core/models/__init__.py b/tools/ui_inspector/core/models/__init__.py deleted file mode 100644 index d957dc7..0000000 --- a/tools/ui_inspector/core/models/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Canonical model exports. - -The compatibility ``models`` package remains importable while callers migrate -to this namespace. -""" - -from models.action import ActionFlow, ActionRecord -from models.element import ElementCatalog, ElementRef -from models.locator import LocatorCandidate -from models.observation import UIObservation -from models.state import UIState - -__all__ = [ - "ActionFlow", "ActionRecord", "ElementCatalog", "ElementRef", - "LocatorCandidate", "UIObservation", "UIState", -] diff --git a/tools/ui_inspector/core/pipeline/__init__.py b/tools/ui_inspector/core/pipeline/__init__.py deleted file mode 100644 index 46f4c6e..0000000 --- a/tools/ui_inspector/core/pipeline/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Shared event, action, and evidence pipeline.""" - -from .event_journal import EventJournal - -__all__ = ["EventJournal"] - diff --git a/tools/ui_inspector/core/pipeline/event_journal.py b/tools/ui_inspector/core/pipeline/event_journal.py deleted file mode 100644 index c6622dd..0000000 --- a/tools/ui_inspector/core/pipeline/event_journal.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Append-only raw interaction journal. - -Raw events are facts. They are persisted before semantic reduction so a -future reducer can rebuild Action Flow without asking the user to repeat the -workflow. -""" - -from __future__ import annotations - -import json -import threading -from dataclasses import asdict, is_dataclass -from datetime import datetime -from pathlib import Path -from typing import Any - -from redaction import sanitize_artifact - - -class EventJournal: - def __init__(self, flow_dir: Path) -> None: - self.path = Path(flow_dir) / "journal" / "raw_events.jsonl" - self.path.parent.mkdir(parents=True, exist_ok=True) - self._lock = threading.Lock() - self._sequence = 0 - - def append(self, event: Any, *, channel: str = "surface") -> int: - if is_dataclass(event) and not isinstance(event, type): - payload = asdict(event) - elif isinstance(event, dict): - payload = dict(event) - else: - payload = dict(getattr(event, "__dict__", {}) or {}) - with self._lock: - self._sequence += 1 - record = { - "journal_sequence": self._sequence, - "received_at": datetime.now().astimezone().isoformat(), - "channel": channel, - "event": sanitize_artifact(payload), - } - line = json.dumps( - record, ensure_ascii=False, sort_keys=True, separators=(",", ":") - ) - with self.path.open("a", encoding="utf-8", newline="\n") as stream: - stream.write(line + "\n") - stream.flush() - return self._sequence - diff --git a/tools/ui_inspector/core/policies/__init__.py b/tools/ui_inspector/core/policies/__init__.py deleted file mode 100644 index 8bf68e9..0000000 --- a/tools/ui_inspector/core/policies/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Surface-independent decision policies.""" - -from .element_identity import element_identity_key -from .target_scoring import resolve_browser_target - -__all__ = ["element_identity_key", "resolve_browser_target"] - diff --git a/tools/ui_inspector/core/policies/element_identity.py b/tools/ui_inspector/core/policies/element_identity.py deleted file mode 100644 index 9adcdc3..0000000 --- a/tools/ui_inspector/core/policies/element_identity.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Stable element identity independent of transient value/focus classes.""" - -from __future__ import annotations - -import hashlib -import json -import re -from typing import Any -from urllib.parse import urlsplit - - -def _origin(url: str) -> str: - try: - parts = urlsplit(url or "") - return f"{parts.scheme}://{parts.netloc}" if parts.netloc else "" - except Exception: - return "" - - -def _structural_css(css: str) -> str: - # Classes frequently encode focus/hover/build hashes. Keep tag depth and - # stable IDs; that distinguishes nested anonymous nodes without freezing a - # generated class list into identity. - parts = [] - for raw in (css or "").split(">"): - part = raw.strip() - tag_match = re.match(r"^[A-Za-z][\w-]*", part) - id_match = re.search(r"#[A-Za-z0-9_-]+", part) - tag = tag_match.group(0).lower() if tag_match else "*" - parts.append(tag + (id_match.group(0) if id_match else "")) - return ">".join(parts) - - -def element_identity_key( - element: dict[str, Any], *, surface: str, page_url: str = "", frame_name: str = "" -) -> str: - el = dict(element or {}) - role = str(el.get("control_type") or el.get("role") or el.get("tag") or "") - anchor = "" - for key in ( - "test_id", "automation_id", "id", "name_attr", "href", - "placeholder", "aria_label", "label", - ): - value = str(el.get(key) or "").strip() - if value: - anchor = f"{key}:{value}" - break - if not anchor and not el.get("contenteditable"): - name = str(el.get("semantic_name") or el.get("name") or "").strip() - if name and len(name) <= 96: - anchor = f"name:{name}" - if not anchor: - anchor = f"structure:{_structural_css(str(el.get('css') or ''))}" - payload = { - "surface": surface or "", - "origin": _origin(page_url) if surface == "browser" else "", - "frame": frame_name or "", - "role": role.lower(), - "anchor": anchor, - } - raw = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) - return hashlib.sha256(raw.encode("utf-8")).hexdigest() diff --git a/tools/ui_inspector/core/policies/target_scoring.py b/tools/ui_inspector/core/policies/target_scoring.py deleted file mode 100644 index 8a0a68f..0000000 --- a/tools/ui_inspector/core/policies/target_scoring.py +++ /dev/null @@ -1,145 +0,0 @@ -"""Distance- and geometry-aware target selection. - -The resolver never relies on a product selector. It ranks the composed path -using native semantics, explicit accessibility metadata, distance from the hit -node, and geometry. Huge ancestors are evidence containers, not controls. -""" - -from __future__ import annotations - -import re -from typing import Any, Iterable - -_NATIVE_TAGS = {"button", "a", "input", "textarea", "select", "summary", "label"} -_INTERACTIVE_ROLES = { - "button", "link", "textbox", "combobox", "checkbox", "radio", - "menuitem", "option", "tab", "switch", "slider", "spinbutton", -} -_ACTION_CLASS_TOKENS = { - "button", "btn", "submit", "send", "publish", "post", "play", - "pause", "action", "click", "toggle", "menu", "option", "tab", -} -_GRAPHIC_LEAF_TAGS = {"path", "use", "g", "svg"} - - -def _tokens(value: str) -> set[str]: - value = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", value or "") - return {x.lower() for x in re.split(r"[^A-Za-z0-9]+", value) if x} - - -def _area(rect: dict[str, Any]) -> float: - try: - return max(0.0, float(rect.get("width") or 0)) * max( - 0.0, float(rect.get("height") or 0) - ) - except (TypeError, ValueError): - return 0.0 - - -def _contains( - rect: dict[str, Any], x: int | None, y: int | None, *, slop: float = 12.0 -) -> bool: - if x is None or y is None or not rect: - return True - try: - return ( - float(rect.get("left")) - slop <= x <= float(rect.get("right")) + slop - and float(rect.get("top")) - slop <= y <= float(rect.get("bottom")) + slop - ) - except (TypeError, ValueError): - return True - - -def resolve_browser_target( - hit_target: dict[str, Any], - candidates: Iterable[dict[str, Any]], - *, - x: int | None = None, - y: int | None = None, -) -> tuple[dict[str, Any], list[dict[str, Any]]]: - """Return the best actionable node and an auditable scored candidate list.""" - chain = [dict(item or {}) for item in candidates if isinstance(item, dict)] - if not chain: - chain = [dict(hit_target or {})] - hit = dict(hit_target or chain[0] or {}) - hit_area = max(1.0, _area(hit.get("rect") or {})) - scored: list[dict[str, Any]] = [] - - for depth, item in enumerate(chain): - tag = str(item.get("tag") or "").lower() - role = str(item.get("role") or "").lower() - explicit_role = str(item.get("explicit_role") or "").lower() - rect = dict(item.get("rect") or {}) - area = _area(rect) - score = 24.0 if depth == 0 else 0.0 - reasons: list[str] = [f"depth={depth}"] - - if tag in _NATIVE_TAGS: - score = max(score, 100.0) - reasons.append("native_control") - if item.get("contenteditable_attr"): - score = max(score, 99.0) - reasons.append("editable_root") - elif item.get("contenteditable"): - score = max(score, 82.0) - reasons.append("editable_descendant") - if explicit_role in _INTERACTIVE_ROLES: - score = max(score, 97.0) - reasons.append("explicit_interactive_role") - elif role in _INTERACTIVE_ROLES: - score = max(score, 88.0) - reasons.append("computed_interactive_role") - if item.get("test_id") or item.get("aria_label"): - score = max(score, 91.0) - reasons.append("stable_semantic_attribute") - if item.get("tabindex") not in (None, ""): - score = max(score, 76.0) - reasons.append("keyboard_focusable") - if _tokens(str(item.get("class_name") or "")) & _ACTION_CLASS_TOKENS: - score = max(score, 82.0) - reasons.append("action_class_token") - if str(item.get("cursor") or "").lower() == "pointer": - score = max(score, 68.0) - reasons.append("pointer_cursor") - - # Target selection and locator quality are one decision. Event-time - # counts are captured before a click can navigate or mutate the DOM. - counts = [ - item.get("stable_css_count"), - item.get("href_count"), - item.get("css_count"), - ] - known_counts = [int(value) for value in counts if value is not None] - if 1 in known_counts: - score += 22.0 - reasons.append("event_time_unique_locator") - elif known_counts and min(known_counts) > 1: - score -= 18.0 - reasons.append("event_time_ambiguous_locator") - - if tag in _GRAPHIC_LEAF_TAGS: - score -= 28.0 - reasons.append("graphic_leaf") - elif ( - str(item.get("cursor") or "").lower() == "pointer" - and 20.0 <= float(rect.get("width") or 0.0) <= 96.0 - and 20.0 <= float(rect.get("height") or 0.0) <= 96.0 - ): - score += 14.0 - reasons.append("compact_clickable_wrapper") - - score -= depth * 3.5 - if not _contains(rect, x, y): - score -= 100.0 - reasons.append("outside_pointer") - if area and area > max(hit_area * 25.0, 200_000.0): - score -= 70.0 - reasons.append("oversized_ancestor") - elif area and area > max(hit_area * 8.0, 60_000.0): - score -= 25.0 - reasons.append("large_ancestor") - - scored.append({"depth": depth, "score": round(score, 3), "reasons": reasons, "target": item}) - - winner = max(scored, key=lambda row: (row["score"], -row["depth"])) - return dict(winner["target"]), scored diff --git a/tools/ui_inspector/desktop_uia.py b/tools/ui_inspector/desktop_uia.py deleted file mode 100644 index ba5f743..0000000 --- a/tools/ui_inspector/desktop_uia.py +++ /dev/null @@ -1,1731 +0,0 @@ -"""UI Automation helpers: attach, control-from-point, tree dump.""" - -from __future__ import annotations - -import ctypes -import time -from ctypes import wintypes -from dataclasses import asdict, dataclass, field -from typing import Any, Optional - -user32 = ctypes.windll.user32 - -WM_GETOBJECT = 0x003D -OBJID_WINDOW = 0x00000000 -OBJID_CLIENT = 0xFFFFFFFC -OBJID_SYSMENU = 0xFFFFFFFF -# UIA root object id ((DWORD)-25) — forces UI Automation provider, not just MSAA. -UIA_ROOT_OBJECT_ID = ctypes.c_long(0xFFFFFFE7).value -SW_RESTORE = 9 -SW_SHOW = 5 -SW_MAXIMIZE = 3 -GA_ROOT = 2 -GW_HWNDNEXT = 2 -GW_OWNER = 4 -GWL_EXSTYLE = -20 -WS_EX_TOOLWINDOW = 0x00000080 -WS_EX_NOACTIVATE = 0x08000000 -DWMWA_CLOAKED = 14 - - -@dataclass -class WindowInfo: - hwnd: int - pid: int - title: str - class_name: str - process_name: str = "" - - def to_dict(self) -> dict[str, Any]: - return asdict(self) - - -@dataclass -class ControlInfo: - name: str = "" - value: str = "" - automation_id: str = "" - control_type: str = "" - class_name: str = "" - framework_id: str = "" - localized_control_type: str = "" - is_enabled: Optional[bool] = None - is_offscreen: Optional[bool] = None - rect: dict[str, int] = field(default_factory=dict) - depth: int = 0 - parent_chain: list[dict[str, str]] = field(default_factory=list) - path_hint: str = "" - - def to_dict(self) -> dict[str, Any]: - return asdict(self) - - -def ensure_uia() -> None: - try: - import uiautomation # noqa: F401 - except ImportError as exc: - raise RuntimeError( - "缺少 uiautomation。请先: pip install uiautomation>=2.0.18" - ) from exc - - -def ensure_com_sta() -> None: - """ - UIAutomation is STA-friendly. Worker threads must enter an apartment - or FindAll often returns an empty shell (findall=2~3). - """ - try: - import comtypes - - try: - comtypes.CoInitializeEx(comtypes.COINIT_APARTMENTTHREADED) - except OSError: - # Already initialized on this thread (or MTA) — ignore. - pass - except Exception: - try: - comtypes.CoInitialize() - except Exception: - pass - except Exception: - pass - - -def window_area(hwnd: int) -> int: - rect = _window_rect(int(hwnd)) - if not rect: - return 0 - return max(0, int(rect.get("width") or 0)) * max(0, int(rect.get("height") or 0)) - - -def _force_foreground(hwnd: int) -> bool: - """AttachThreadInput + SetForeground — more reliable than bare SetForeground.""" - hwnd = int(hwnd) - try: - fg = int(user32.GetForegroundWindow() or 0) - tid_fg = int(user32.GetWindowThreadProcessId(fg, None) or 0) if fg else 0 - tid_t = int(user32.GetWindowThreadProcessId(hwnd, None) or 0) - attached = False - if tid_fg and tid_t and tid_fg != tid_t: - attached = bool(user32.AttachThreadInput(tid_fg, tid_t, True)) - try: - user32.BringWindowToTop(hwnd) - ok = bool(user32.SetForegroundWindow(hwnd)) - try: - user32.SetFocus(hwnd) - except Exception: - pass - return ok - finally: - if attached: - user32.AttachThreadInput(tid_fg, tid_t, False) - except Exception: - try: - return bool(user32.SetForegroundWindow(hwnd)) - except Exception: - return False - - -def activate_main_hwnd(hwnd: int, *, maximize: bool = False) -> dict[str, Any]: - """ - Light activation for capture: - - restore only if minimized - - maximize only when explicitly requested (Inspector default: off) - - SetForeground only if not already foreground (avoid flash) - Only call on the chosen main/capture hwnd — never on every child. - """ - hwnd = int(hwnd) - info: dict[str, Any] = { - "hwnd": hwnd, - "iconic": False, - "zoomed_before": False, - "maximized": False, - "foreground": False, - "already_foreground": False, - "maximize_requested": bool(maximize), - } - try: - info["iconic"] = bool(user32.IsIconic(hwnd)) - info["zoomed_before"] = bool(user32.IsZoomed(hwnd)) - except Exception: - pass - - if info["iconic"]: - user32.ShowWindow(hwnd, SW_RESTORE) - time.sleep(0.15) - if maximize and not info["zoomed_before"]: - user32.ShowWindow(hwnd, SW_MAXIMIZE) - time.sleep(0.2) - - try: - already = int(user32.GetForegroundWindow() or 0) == hwnd - except Exception: - already = False - info["already_foreground"] = already - if already: - info["foreground"] = True - else: - info["foreground"] = _force_foreground(hwnd) - time.sleep(0.25 if maximize else 0.15) - try: - info["maximized"] = bool(user32.IsZoomed(hwnd)) - except Exception: - pass - return info - - -def is_provider_cold(hwnd: int, *, findall: Optional[int] = None) -> bool: - """True when WM_GETOBJECT CLIENT is 0 and FindAll is an empty shell.""" - hwnd = int(hwnd) - n = int(findall) if findall is not None else findall_count_for_hwnd(hwnd) - if n < 0: - n = 0 - if n >= 8: - return False - try: - res = int(user32.SendMessageW(hwnd, WM_GETOBJECT, 0, OBJID_CLIENT) or 0) - except Exception: - res = 0 - return res == 0 and n < 8 - - -def is_mainish_window(info: WindowInfo) -> bool: - """Large primary app window (safe to maximize).""" - cls = info.class_name or "" - if cls in {"SearchResultWindow2", "IME", "MSCTFIME UI", "tooltips_class32"}: - return False - if "SearchResult" in cls: - return False - area = window_area(info.hwnd) - if area >= 500 * 400: - return True - if "WeWork" in cls or "Chrome_WidgetWin" in cls: - return area >= 300 * 200 - return bool((info.title or "").strip()) and area >= 600 * 400 - - -def largest_window(candidates: list[WindowInfo]) -> Optional[WindowInfo]: - if not candidates: - return None - return max(candidates, key=lambda c: window_area(c.hwnd)) - - -def _safe_str(value: Any) -> str: - if value is None: - return "" - try: - return str(value).strip() - except Exception: - return "" - - -def _rect_dict(ctrl) -> dict[str, int]: - try: - r = ctrl.BoundingRectangle - return { - "left": int(r.left), - "top": int(r.top), - "right": int(r.right), - "bottom": int(r.bottom), - "width": int(r.width()), - "height": int(r.height()), - } - except Exception: - return {} - - -def _read_value(ctrl) -> str: - # Align with production skills: Value often lives in ValueValueProperty. - try: - import uiautomation as auto - - raw = ctrl.GetPropertyValue(auto.PropertyId.ValueValueProperty) - s = _safe_str(raw) - if s: - return s - except Exception: - pass - try: - pattern = ctrl.GetValuePattern() - if pattern is not None: - s = _safe_str(pattern.Value) - if s: - return s - except Exception: - pass - for attr in ("Value", "LegacyIAccessibleValue", "HelpText"): - try: - s = _safe_str(getattr(ctrl, attr, None)) - if s: - return s - except Exception: - continue - return "" - - -def _control_type_name(ctrl) -> str: - try: - ct = ctrl.ControlTypeName - if ct: - return _safe_str(ct) - except Exception: - pass - try: - return _safe_str(ctrl.ControlType) - except Exception: - return "" - - -def describe_control(ctrl, *, depth: int = 0, parent_chain: Optional[list] = None) -> ControlInfo: - info = ControlInfo( - name=_safe_str(getattr(ctrl, "Name", None)), - value=_read_value(ctrl), - automation_id=_safe_str(getattr(ctrl, "AutomationId", None)), - control_type=_control_type_name(ctrl), - class_name=_safe_str(getattr(ctrl, "ClassName", None)), - framework_id=_safe_str(getattr(ctrl, "FrameworkId", None)), - localized_control_type=_safe_str(getattr(ctrl, "LocalizedControlType", None)), - depth=depth, - parent_chain=list(parent_chain or []), - rect=_rect_dict(ctrl), - ) - try: - info.is_enabled = bool(ctrl.IsEnabled) - except Exception: - info.is_enabled = None - try: - info.is_offscreen = bool(ctrl.IsOffscreen) - except Exception: - info.is_offscreen = None - - bits = [] - if info.control_type: - bits.append(info.control_type) - if info.name: - bits.append(f"Name={info.name!r}") - if info.value: - bits.append(f"Value={info.value!r}") - if info.automation_id: - bits.append(f"AutomationId={info.automation_id!r}") - info.path_hint = " | ".join(bits) - return info - - -def get_toplevel_hwnd(hwnd: int) -> int: - hwnd = int(hwnd) - if not hwnd: - return 0 - root = int(user32.GetAncestor(hwnd, GA_ROOT) or 0) - return root or hwnd - - -def window_info_from_hwnd(hwnd: int) -> WindowInfo: - ensure_uia() - import uiautomation as auto - - hwnd = get_toplevel_hwnd(int(hwnd)) - if not hwnd or not user32.IsWindow(hwnd): - raise RuntimeError(f"无效窗口 HWND={hwnd}") - - title_buf = ctypes.create_unicode_buffer(512) - user32.GetWindowTextW(hwnd, title_buf, 512) - class_buf = ctypes.create_unicode_buffer(256) - user32.GetClassNameW(hwnd, class_buf, 256) - - pid = ctypes.c_ulong(0) - user32.GetWindowThreadProcessId(hwnd, ctypes.byref(pid)) - - process_name = "" - try: - root = auto.ControlFromHandle(hwnd) - process_name = _safe_str(getattr(root, "ProcessName", None)) - except Exception: - pass - - return WindowInfo( - hwnd=hwnd, - pid=int(pid.value), - title=_safe_str(title_buf.value), - class_name=_safe_str(class_buf.value), - process_name=process_name, - ) - - -def get_foreground_window_info() -> WindowInfo: - hwnd = int(user32.GetForegroundWindow()) - if not hwnd: - raise RuntimeError("当前没有前台窗口") - return window_info_from_hwnd(hwnd) - - -def is_inspector_window(info: WindowInfo, *, self_hwnd: int = 0) -> bool: - title = info.title or "" - if self_hwnd and int(info.hwnd) == int(self_hwnd): - return True - if self_hwnd and get_toplevel_hwnd(self_hwnd) == int(info.hwnd): - return True - if "UI Inspector" in title or "OpenClaw UI Inspector" in title: - return True - return False - - -def is_system_chrome_window(info: WindowInfo) -> bool: - cls = info.class_name or "" - return cls in { - "Shell_TrayWnd", - "Shell_SecondaryTrayWnd", - "NotifyIconOverflowWindow", - "Progman", - "WorkerW", - } - - -def _window_rect(hwnd: int) -> dict[str, int]: - rect = wintypes.RECT() - if not user32.GetWindowRect(int(hwnd), ctypes.byref(rect)): - return {} - return { - "left": int(rect.left), - "top": int(rect.top), - "right": int(rect.right), - "bottom": int(rect.bottom), - "width": int(rect.right - rect.left), - "height": int(rect.bottom - rect.top), - } - - -def _is_cloaked(hwnd: int) -> bool: - try: - dwmapi = ctypes.windll.dwmapi - cloaked = ctypes.c_int(0) - hr = dwmapi.DwmGetWindowAttribute( - wintypes.HWND(int(hwnd)), - ctypes.c_int(DWMWA_CLOAKED), - ctypes.byref(cloaked), - ctypes.sizeof(cloaked), - ) - return hr == 0 and int(cloaked.value) != 0 - except Exception: - return False - - -def _exstyle(hwnd: int) -> int: - return int(user32.GetWindowLongW(int(hwnd), GWL_EXSTYLE) or 0) - - -def iter_toplevel_hwnds_zorder(): - """Front-to-back top-level windows (generic Win32 Z-order walk).""" - hwnd = int(user32.GetTopWindow(None) or 0) - seen: set[int] = set() - while hwnd: - if hwnd in seen: - break - seen.add(hwnd) - yield hwnd - hwnd = int(user32.GetWindow(hwnd, GW_HWNDNEXT) or 0) - - -def list_capturable_candidates( - *, - exclude_hwnds: Optional[set[int]] = None, - min_width: int = 120, - min_height: int = 80, -) -> list[WindowInfo]: - """Z-order capturable windows; no app-name hardcoding.""" - exclude = {int(h) for h in (exclude_hwnds or set()) if h} - # Also exclude top-level of each excluded hwnd - exclude |= {get_toplevel_hwnd(h) for h in list(exclude)} - - out: list[WindowInfo] = [] - for hwnd in iter_toplevel_hwnds_zorder(): - if hwnd in exclude: - continue - if not user32.IsWindowVisible(hwnd): - continue - if user32.IsIconic(hwnd): - continue - if _is_cloaked(hwnd): - continue - rect = _window_rect(hwnd) - if not rect or rect.get("width", 0) < min_width or rect.get("height", 0) < min_height: - continue - try: - info = window_info_from_hwnd(hwnd) - except Exception: - continue - if "UI Inspector" in (info.title or "") or "OpenClaw UI Inspector" in (info.title or ""): - continue - if is_system_chrome_window(info) and info.class_name in { - "Shell_TrayWnd", - "Shell_SecondaryTrayWnd", - "NotifyIconOverflowWindow", - "Progman", - "WorkerW", - }: - continue - if is_system_chrome_window(info) and not (info.title or "").strip(): - continue - # Skip untitled toolwindows (IME, ghosts) - ex = _exstyle(hwnd) - if (ex & WS_EX_TOOLWINDOW) and not (info.title or "").strip(): - continue - out.append(info) - return out - - -def _owner_root(hwnd: int) -> int: - cur = int(hwnd) - seen: set[int] = set() - while cur and cur not in seen: - seen.add(cur) - owner = int(user32.GetWindow(cur, GW_OWNER) or 0) - if not owner: - break - cur = owner - return cur or int(hwnd) - - -def prefer_last_active_popup(info: WindowInfo) -> WindowInfo: - """If Win32 reports an active owned popup, prefer it (still app-agnostic).""" - try: - root = _owner_root(info.hwnd) - popup = int(user32.GetLastActivePopup(root) or 0) - if not popup or popup == int(info.hwnd): - # also try popup of the candidate itself - popup = int(user32.GetLastActivePopup(int(info.hwnd)) or 0) - if not popup or not user32.IsWindowVisible(popup) or user32.IsIconic(popup): - return info - if _is_cloaked(popup): - return info - prect = _window_rect(popup) - if not prect or prect.get("width", 0) < 80 or prect.get("height", 0) < 60: - return info - popup_info = window_info_from_hwnd(popup) - if is_system_chrome_window(popup_info): - return info - return popup_info - except Exception: - return info - - -def window_info_under_cursor( - *, - exclude_hwnds: Optional[set[int]] = None, -) -> Optional[WindowInfo]: - """光标下的顶层窗（不 Activate、不点击）。光标在 Inspector 上则返回 None。""" - exclude = {int(h) for h in (exclude_hwnds or set()) if h} - exclude |= {get_toplevel_hwnd(h) for h in list(exclude)} - pt = wintypes.POINT() - if not user32.GetCursorPos(ctypes.byref(pt)): - return None - hwnd = int(user32.WindowFromPoint(pt) or 0) - if not hwnd: - return None - top = get_toplevel_hwnd(hwnd) - if not top or top in exclude: - return None - if not user32.IsWindowVisible(top) or user32.IsIconic(top) or _is_cloaked(top): - return None - try: - info = window_info_from_hwnd(top) - except Exception: - return None - if is_inspector_window(info): - return None - if "UI Inspector" in (info.title or "") or "OpenClaw UI Inspector" in (info.title or ""): - return None - return info - - -def find_wecom_search_result_window( - *, - pid: Optional[int] = None, - exclude_hwnds: Optional[set[int]] = None, -) -> Optional[WindowInfo]: - """同进程可见的 SearchResultWindow2（搜索联系人浮层），不要求前台焦点。""" - exclude = {int(h) for h in (exclude_hwnds or set()) if h} - exclude |= {get_toplevel_hwnd(h) for h in list(exclude)} - best: Optional[WindowInfo] = None - best_area = -1 - for hwnd in iter_toplevel_hwnds_zorder(): - if hwnd in exclude: - continue - if not user32.IsWindowVisible(hwnd) or user32.IsIconic(hwnd) or _is_cloaked(hwnd): - continue - cls_buf = ctypes.create_unicode_buffer(256) - user32.GetClassNameW(hwnd, cls_buf, 256) - if (cls_buf.value or "") != "SearchResultWindow2": - continue - try: - info = window_info_from_hwnd(hwnd) - except Exception: - continue - if pid is not None and int(info.pid) != int(pid): - continue - rect = _window_rect(hwnd) or {} - area = int(rect.get("width") or 0) * int(rect.get("height") or 0) - if area < 40 * 40: - continue - if area > best_area: - best_area = area - best = info - return best - - -def foreground_toplevel_hwnd(*, exclude_hwnds: Optional[set[int]] = None) -> Optional[int]: - """True foreground top-level hwnd (GetForegroundWindow), if capturable.""" - exclude = {int(h) for h in (exclude_hwnds or set()) if h} - exclude |= {get_toplevel_hwnd(h) for h in list(exclude)} - try: - fg = int(user32.GetForegroundWindow() or 0) - except Exception: - fg = 0 - if not fg: - return None - top = get_toplevel_hwnd(fg) - if top in exclude: - return None - if not user32.IsWindowVisible(top) or user32.IsIconic(top) or _is_cloaked(top): - return None - return int(top) - - -def find_capture_target( - *, - exclude_hwnds: Optional[set[int]] = None, - prefer_pid: Optional[int] = None, - under_cursor: bool = False, - prefer_wecom_search_popup: bool = True, -) -> tuple[WindowInfo, list[WindowInfo]]: - """ - Pick a capturable top-level window. - - Default (under_cursor=False) keeps legacy behaviour: - 1) GetForegroundWindow, if capturable - 2) else Z-order top capturable candidate - 3) prefer_last_active_popup (same pid) - - Optional: - - under_cursor: 光标下顶层窗优先（不点击、不抢焦点）；失败则回退前台逻辑 - - prefer_wecom_search_popup: 若候选为企微主窗且存在 SearchResultWindow2，改采浮层 - """ - info, cands, _meta = resolve_capture_target( - exclude_hwnds=exclude_hwnds, - prefer_pid=prefer_pid, - under_cursor=under_cursor, - prefer_wecom_search_popup=prefer_wecom_search_popup, - ) - return info, cands - - -def resolve_capture_target( - *, - exclude_hwnds: Optional[set[int]] = None, - prefer_pid: Optional[int] = None, - under_cursor: bool = False, - prefer_wecom_search_popup: bool = True, -) -> tuple[WindowInfo, list[WindowInfo], dict[str, Any]]: - """与 find_capture_target 相同，额外返回 selection meta（strategy 链）。""" - candidates = list_capturable_candidates(exclude_hwnds=exclude_hwnds) - if not candidates: - raise RuntimeError( - "未找到可拾取的顶层窗口。请确认目标软件窗口可见且未被最小化。" - ) - - by_hwnd = {int(c.hwnd): c for c in candidates} - chosen: Optional[WindowInfo] = None - strategy_parts: list[str] = [] - - if under_cursor: - cur = window_info_under_cursor(exclude_hwnds=exclude_hwnds) - if cur is not None: - chosen = cur - strategy_parts.append("under_cursor") - - if chosen is None: - fg = foreground_toplevel_hwnd(exclude_hwnds=exclude_hwnds) - if fg and fg in by_hwnd: - # Frontmost always wins — never replace with a larger/sticky other app. - chosen = by_hwnd[fg] - strategy_parts.append("foreground") - elif fg: - # Foreground exists but filtered from candidates — still try to describe it. - try: - info = window_info_from_hwnd(fg) - if not is_inspector_window(info) and not ( - is_system_chrome_window(info) and not (info.title or "").strip() - ): - chosen = info - strategy_parts.append("foreground_raw") - except Exception: - chosen = None - - if chosen is None: - # No usable foreground: Z-order front. prefer_pid only as soft filter here. - if prefer_pid: - for c in candidates: - if int(c.pid) == int(prefer_pid): - chosen = c - strategy_parts.append("sticky_pid") - break - if chosen is None: - chosen = candidates[0] - strategy_parts.append("zorder") - - # Owned popup of the *same* front window only (e.g. search panel), never another app - popup = prefer_last_active_popup(chosen) - if int(popup.pid) == int(chosen.pid) and int(popup.hwnd) != int(chosen.hwnd): - chosen = popup - strategy_parts.append("last_active_popup") - - # 企微：主窗为前台时，若搜索浮层仍打开，优先采浮层（无需点击进会话） - if prefer_wecom_search_popup and (chosen.class_name or "") == "WeWorkWindow": - search_win = find_wecom_search_result_window( - pid=int(chosen.pid), exclude_hwnds=exclude_hwnds - ) - if search_win is not None: - chosen = search_win - strategy_parts.append("wecom_search_popup") - - meta = { - "strategy": "+".join(strategy_parts) if strategy_parts else "unknown", - "under_cursor": bool(under_cursor), - "prefer_wecom_search_popup": bool(prefer_wecom_search_popup), - "chosen_class": chosen.class_name, - "chosen_hwnd": int(chosen.hwnd), - } - return chosen, candidates, meta - - -def wait_frontmost_target( - *, - exclude_hwnds: Optional[set[int]] = None, - timeout_sec: float = 12.0, - poll_sec: float = 0.35, -) -> tuple[WindowInfo, list[WindowInfo], dict[str, Any]]: - """ - Wait until the true foreground window is a capturable target - (not the Inspector). Frontmost = the capture target. No size ranking. - """ - ensure_com_sta() - ensure_uia() - exclude = {int(h) for h in (exclude_hwnds or set()) if h} - deadline = time.time() + float(timeout_sec) - meta: dict[str, Any] = {"waited": False, "polls": 0} - last_err = "" - while time.time() < deadline: - meta["polls"] += 1 - try: - info, cands = find_capture_target(exclude_hwnds=exclude_hwnds) - except Exception as exc: - last_err = str(exc) - time.sleep(poll_sec) - continue - # If foreground resolved to an excluded hwnd path, candidates[0] is next. - # Reject only when we somehow still landed on Inspector. - if is_inspector_window(info): - time.sleep(poll_sec) - meta["waited"] = True - continue - meta["window"] = info.to_dict() - return info, cands, meta - if last_err: - raise RuntimeError(f"等待最前端目标窗口超时：{last_err}") - raise RuntimeError( - "等待最前端目标窗口超时：请先用鼠标点一下要采集的软件窗口（不要点 Inspector）。" - ) - - -def prepare_frontmost_for_capture( - *, - exclude_hwnds: Optional[set[int]] = None, - timeout_sec: float = 12.0, -) -> dict[str, Any]: - """ - Resolve frontmost target, light-activate if mainish (no force maximize), warmup. - Always returns the frontmost window — findall is informational only, - never a reason to pick another app. - - 企微：先静默调用 kit ``ensure_ui_ready("wecom")``（与技能同路径），再轻量 poke。 - """ - info, cands, wait_meta = wait_frontmost_target( - exclude_hwnds=exclude_hwnds, timeout_sec=timeout_sec - ) - act = None - if is_mainish_window(info): - # Keep operator size/position; only restore if minimized. - act = activate_main_hwnd(info.hwnd, maximize=False) - - # Product adapters may need to make their accessibility provider ready - # before the operator starts typing. Running this on the app's existing - # startup worker keeps Tk responsive and prevents the first input - # transaction from racing a 10-20 second background warmup. - import kit_wake - - kit_report = kit_wake.auto_ensure_if_wecom( - class_name=info.class_name or "", - title=info.title or "", - hwnd=info.hwnd, - ) - findall = kit_report.get("findall") if kit_report.get("ok") else None - provider_cold = bool(kit_report.get("applicable") and not kit_report.get("ok")) - - return { - "ready": True, # frontmost resolved => ready to capture that window - "window": info.to_dict(), - "candidates": len(cands), - "findall": findall, - "uia_sparse": bool(provider_cold), - "provider_cold": provider_cold, - "activate": act, - "warmup": {"mode": "startup_adapter" if kit_report.get("applicable") else "deferred"}, - "kit_wake": kit_report, - "wait": wait_meta, - "strategy": "foreground_only", - } - - -def enum_child_hwnds(parent: int, *, max_count: int = 80) -> list[int]: - result: list[int] = [] - - @ctypes.WINFUNCTYPE(ctypes.c_bool, wintypes.HWND, wintypes.LPARAM) - def _cb(hwnd, _lparam): - if len(result) >= max_count: - return False - if user32.IsWindowVisible(hwnd): - result.append(int(hwnd)) - return True - - user32.EnumChildWindows(int(parent), _cb, 0) - return result - - -def is_window_minimized(hwnd: int) -> bool: - hwnd = int(hwnd) - if user32.IsIconic(hwnd): - return True - rect = _window_rect(hwnd) - # Windows parks minimized windows near (-32000,-32000) - if rect and int(rect.get("left") or 0) <= -30000 and int(rect.get("top") or 0) <= -30000: - return True - return False - - -def restore_window_if_needed( - hwnd: int, - *, - force_show: bool = False, - bring_foreground: bool = False, -) -> dict[str, Any]: - """ - Only restore when minimized by default. - - Avoid unconditional ShowWindow/SetForeground — repeating those on child/tool - HWNDs (esp. WeCom) spawns/flickers many tiny windows under the process. - """ - hwnd = int(hwnd) - info = { - "hwnd": hwnd, - "was_minimized": False, - "restored": False, - "shown": False, - "foreground": False, - } - if is_window_minimized(hwnd): - info["was_minimized"] = True - user32.ShowWindow(hwnd, SW_RESTORE) - time.sleep(0.2) - info["restored"] = True - elif force_show: - try: - user32.ShowWindow(hwnd, SW_SHOW) - info["shown"] = True - except Exception: - pass - if bring_foreground: - try: - if user32.SetForegroundWindow(hwnd): - info["foreground"] = True - except Exception: - pass - time.sleep(0.05) - return info - - -def findall_count_for_hwnd(hwnd: int) -> int: - try: - from uiautomation.uiautomation import _AutomationClient - - root = control_from_hwnd(int(hwnd)) - ac = _AutomationClient.instance() - arr = root.Element.FindAll(7, ac.IUIAutomation.CreateTrueCondition()) - return int(arr.Length) - except Exception: - return -1 - - -def poke_uia_provider(hwnd: int) -> dict[str, Any]: - """ - Light provider poke: no ShowWindow / no SetForeground. - - Safe to call on child or sibling HWNDs without popping tool windows. - """ - ensure_uia() - hwnd = int(hwnd) - report: dict[str, Any] = {"hwnd": hwnd, "mode": "light", "steps": []} - for obj_id, label in ( - (UIA_ROOT_OBJECT_ID, "UiaRootObjectId"), - (OBJID_CLIENT, "OBJID_CLIENT"), - ): - try: - res = int(user32.SendMessageW(hwnd, WM_GETOBJECT, 0, obj_id) or 0) - report["steps"].append({"wm_getobject": label, "lresult": res}) - except Exception as exc: - report["steps"].append({"wm_getobject": label, "error": str(exc)}) - try: - from uiautomation.uiautomation import _AutomationClient - import uiautomation as auto - - ac = _AutomationClient.instance() - el = ac.IUIAutomation.ElementFromHandle(wintypes.HWND(hwnd)) - report["steps"].append({"ElementFromHandle": el is not None}) - ctrl = auto.ControlFromHandle(hwnd) - try: - arr = ctrl.Element.FindAll(4, ac.IUIAutomation.CreateTrueCondition()) - report["steps"].append({"poke_FindAll_descendants": int(arr.Length)}) - except Exception as exc: - report["steps"].append({"poke_FindAll": str(exc)}) - except Exception as exc: - report["steps"].append({"uia_refresh": str(exc)}) - return report - - -def _msaa_and_uia_poke(hwnd: int, report: dict[str, Any]) -> None: - """Shared MSAA/UIA attach steps (no ShowWindow).""" - hwnd = int(hwnd) - for obj_id, label in ( - (UIA_ROOT_OBJECT_ID, "UiaRootObjectId"), - (OBJID_CLIENT, "OBJID_CLIENT"), - (OBJID_WINDOW, "OBJID_WINDOW"), - ): - try: - res = int(user32.SendMessageW(hwnd, WM_GETOBJECT, 0, obj_id) or 0) - report["steps"].append({"wm_getobject": label, "lresult": res}) - except Exception as exc: - report["steps"].append({"wm_getobject": label, "error": str(exc)}) - - try: - from comtypes import GUID, IUnknown - - iid = GUID("{618736e0-3c3d-11cf-810c-00aa00389b71}") # IID_IAccessible - punk = ctypes.c_void_p() - hr = ctypes.windll.oleacc.AccessibleObjectFromWindow( - wintypes.HWND(hwnd), - ctypes.c_long(OBJID_CLIENT), - ctypes.byref(iid), - ctypes.byref(punk), - ) - report["steps"].append( - { - "AccessibleObjectFromWindow": int(hr), - "has_punk": bool(punk.value), - } - ) - if punk.value: - try: - ctypes.cast(punk, ctypes.POINTER(IUnknown)).Release() - except Exception: - pass - except Exception as exc: - report["steps"].append({"AccessibleObjectFromWindow": str(exc)}) - - try: - from uiautomation.uiautomation import _AutomationClient - import uiautomation as auto - - ac = _AutomationClient.instance() - el = ac.IUIAutomation.ElementFromHandle(wintypes.HWND(hwnd)) - report["steps"].append({"ElementFromHandle": el is not None}) - ctrl = auto.ControlFromHandle(hwnd) - try: - kids = ctrl.GetChildren() - report["steps"].append({"GetChildren_count": len(kids or [])}) - except Exception as exc: - report["steps"].append({"GetChildren": str(exc)}) - try: - arr = ctrl.Element.FindAll(4, ac.IUIAutomation.CreateTrueCondition()) - report["steps"].append({"poke_FindAll_descendants": int(arr.Length)}) - except Exception as exc: - report["steps"].append({"poke_FindAll": str(exc)}) - except Exception as exc: - report["steps"].append({"uia_refresh": str(exc)}) - - -def warmup_uia_provider( - hwnd: int, - *, - interact_window: bool = True, - allow_foreground_nudge: bool = True, - sparse_findall_threshold: int = 8, -) -> dict[str, Any]: - """ - Force accessibility provider initialization (Inspect-like). - - - Restores minimized main window only (no blanket Show on children). - - If still sparse after poke, optionally SetForeground once on THIS hwnd - (never on child/tool HWNDs — that was the popup storm). - """ - ensure_uia() - hwnd = int(hwnd) - report: dict[str, Any] = {"hwnd": hwnd, "mode": "full", "steps": []} - - if interact_window: - report["restore"] = restore_window_if_needed( - hwnd, force_show=False, bring_foreground=False - ) - else: - report["restore"] = {"skipped": True} - - report["findall_before"] = findall_count_for_hwnd(hwnd) - _msaa_and_uia_poke(hwnd, report) - time.sleep(0.15) - report["findall_after"] = findall_count_for_hwnd(hwnd) - - if ( - allow_foreground_nudge - and interact_window - and int(report.get("findall_after") or 0) < int(sparse_findall_threshold) - ): - # One-shot focus nudge on the capture target only. - nudge = restore_window_if_needed( - hwnd, force_show=False, bring_foreground=True - ) - report["foreground_nudge"] = nudge - time.sleep(0.35) - _msaa_and_uia_poke(hwnd, report) - report["findall_after_nudge"] = findall_count_for_hwnd(hwnd) - report["findall_after"] = report["findall_after_nudge"] - - report["woke"] = int(report.get("findall_after") or 0) > max( - 5, int(report.get("findall_before") or 0) + 2 - ) - report["provider_cold"] = is_provider_cold( - hwnd, findall=int(report.get("findall_after") or 0) - ) - return report - - -def list_same_pid_toplevel_hwnds(pid: int) -> list[int]: - found: list[int] = [] - - @ctypes.WINFUNCTYPE(ctypes.c_bool, wintypes.HWND, wintypes.LPARAM) - def _cb(hwnd, _lp): - p = ctypes.c_ulong(0) - user32.GetWindowThreadProcessId(hwnd, ctypes.byref(p)) - if int(p.value) == int(pid): - found.append(int(hwnd)) - return True - - user32.EnumWindows(_cb, 0) - return found - - -def dump_window_tree( - hwnd: int, - *, - max_depth: int = 16, - max_nodes: int = 8000, - only_visible: bool = False, - enrich_child_hwnds: bool = True, - use_findall: bool = True, - warmup: bool = True, - warmup_mode: str = "full", - merge_same_pid_if_sparse: bool = True, -) -> tuple[dict[str, Any], list[dict[str, Any]], dict[str, Any], list[dict[str, Any]]]: - """ - Dump UIA tree + flat index + locatable candidates. - - warmup_mode: - - full: MSAA/UIA poke on target; restore only if minimized - - light: ElementFromHandle poke only (no ShowWindow) - - probe: light poke + FindAll, but never invoke a blocking product wake - - off: no provider warmup - Child / same-PID merges never call ShowWindow (light poke at most). - """ - import kit_wake - - hwnd = int(hwnd) - mode = (warmup_mode or "full").lower() - if not warmup: - mode = "off" - if mode not in {"full", "light", "probe", "off"}: - mode = "full" - - kit_report: dict[str, Any] = {"applicable": False, "skipped": True, "reason": "not_checked"} - try: - winfo_for_kit = window_info_from_hwnd(hwnd) - except Exception: - winfo_for_kit = None - # 搜索浮层稀疏时勿合并主窗会话列表，否则验收会误判「已有数据」 - allow_same_pid_merge = bool(merge_same_pid_if_sparse) - if winfo_for_kit is not None and (winfo_for_kit.class_name or "") == "SearchResultWindow2": - allow_same_pid_merge = False - if winfo_for_kit is not None and mode not in {"off", "probe"}: - hint = findall_count_for_hwnd(hwnd) - kit_report = kit_wake.auto_ensure_if_wecom( - class_name=winfo_for_kit.class_name or "", - title=winfo_for_kit.title or "", - hwnd=hwnd, - force=False, - findall_hint=hint, - ) - - warmup_report: dict[str, Any] = {"mode": mode} - if mode == "full": - try: - winfo = window_info_from_hwnd(hwnd) - except Exception: - winfo = None - if winfo is not None and is_mainish_window(winfo): - # No force maximize — preserve operator window geometry. - act = activate_main_hwnd(hwnd, maximize=False) - warmup_report = warmup_uia_provider( - hwnd, interact_window=False, allow_foreground_nudge=False - ) - warmup_report["activate"] = act - else: - warmup_report = warmup_uia_provider( - hwnd, interact_window=True, allow_foreground_nudge=True - ) - elif mode in {"light", "probe"}: - warmup_report = poke_uia_provider(hwnd) - - def _run_dump() -> tuple[dict[str, Any], list[dict[str, Any]], dict[str, Any]]: - root_ctrl = control_from_hwnd(hwnd) - t, idx, st = dump_tree( - root_ctrl, - max_depth=max_depth, - max_nodes=max_nodes, - only_visible=only_visible, - ) - st = dict(st) - st["enriched_child_hwnds"] = False - st["findall_count"] = 0 - st["merged_same_pid_hwnds"] = [] - - if enrich_child_hwnds: - children_nodes: list[dict[str, Any]] = list(t.get("children") or []) - seen_hwnd = {int(hwnd)} - for child_hwnd in enum_child_hwnds(hwnd, max_count=80): - if child_hwnd in seen_hwnd: - continue - seen_hwnd.add(child_hwnd) - if int(st.get("nodes_kept") or 0) >= max_nodes: - st["truncated"] = True - break - try: - # Never full-warmup / ShowWindow child HWNDs. - child_root = control_from_hwnd(child_hwnd) - except Exception: - continue - remain = max_nodes - int(st.get("nodes_kept") or 0) - sub_tree, sub_index, sub_stats = dump_tree( - child_root, - max_depth=max(2, max_depth - 1), - max_nodes=max(1, remain), - only_visible=only_visible, - ) - if int(sub_stats.get("nodes_kept") or 0) <= 1 and not ( - (sub_tree.get("name") or "") or (sub_tree.get("value") or "") - ): - if int(sub_stats.get("nodes_kept") or 0) <= 0: - continue - children_nodes.append(sub_tree) - idx.extend(sub_index) - st["nodes_kept"] = int(st.get("nodes_kept") or 0) + int( - sub_stats.get("nodes_kept") or 0 - ) - st["nodes_visited"] = int(st.get("nodes_visited") or 0) + int( - sub_stats.get("nodes_visited") or 0 - ) - st["enriched_child_hwnds"] = True - if sub_stats.get("truncated"): - st["truncated"] = True - if children_nodes: - t = dict(t) - t["children"] = children_nodes - - if use_findall and mode != "off": - found = collect_findall_flat(root_ctrl, max_nodes=max_nodes) - st["findall_count"] = len(found) - existing = {r.get("path_hint") for r in idx} - for row in found: - if row.get("path_hint") in existing: - continue - idx.append(row) - existing.add(row.get("path_hint")) - return t, idx, st - - tree, index, stats = _run_dump() - if winfo_for_kit is not None: - kit_wake.note_provider_warm( - class_name=winfo_for_kit.class_name or "", - title=winfo_for_kit.title or "", - hwnd=hwnd, - findall=int(stats.get("findall_count") or 0), - ) - stats["warmup"] = warmup_report - stats["warmup_mode"] = mode - stats["kit_wake"] = kit_report - stats["walker"] = "GetFirstChild/NextSibling (package default RawViewWalker)" - stats["escalated_warmup"] = False - - locatable = locatable_from_index(index) - stats["locatable_count"] = len(locatable) - stats["uia_sparse"] = int(stats.get("locatable_count") or 0) < 8 - stats["provider_cold"] = is_provider_cold( - hwnd, findall=int(stats.get("findall_count") or 0) - ) - - # 仍冷且目标为企微：强制再 ensure 一次（绕过短时 cache）后重 dump - if mode not in {"off", "probe"} and stats["provider_cold"] and winfo_for_kit is not None and kit_wake.looks_like_wecom( - class_name=winfo_for_kit.class_name or "", - title=winfo_for_kit.title or "", - ): - kit_report = kit_wake.auto_ensure_if_wecom( - class_name=winfo_for_kit.class_name or "", - title=winfo_for_kit.title or "", - hwnd=hwnd, - force=True, - ) - stats["kit_wake"] = kit_report - if kit_report.get("ok"): - tree, index, stats = _run_dump() - stats["warmup"] = warmup_report - stats["warmup_mode"] = mode - stats["kit_wake"] = kit_report - stats["walker"] = "GetFirstChild/NextSibling (package default RawViewWalker)" - stats["escalated_warmup"] = False - stats["kit_wake_retry"] = True - locatable = locatable_from_index(index) - stats["locatable_count"] = len(locatable) - stats["uia_sparse"] = int(stats.get("locatable_count") or 0) < 8 - stats["provider_cold"] = is_provider_cold( - hwnd, findall=int(stats.get("findall_count") or 0) - ) - - # If light dump is sparse, escalate once to full warmup + re-dump (no child ShowWindow). - if stats["uia_sparse"] and mode == "light": - try: - winfo = window_info_from_hwnd(hwnd) - except Exception: - winfo = None - if winfo is not None and is_mainish_window(winfo): - act = activate_main_hwnd(hwnd, maximize=False) - warmup_report = warmup_uia_provider( - hwnd, interact_window=False, allow_foreground_nudge=False - ) - warmup_report["activate"] = act - else: - warmup_report = warmup_uia_provider( - hwnd, interact_window=True, allow_foreground_nudge=True - ) - tree, index, stats = _run_dump() - stats["warmup"] = warmup_report - stats["warmup_mode"] = "full" - stats["kit_wake"] = kit_report - stats["escalated_warmup"] = True - locatable = locatable_from_index(index) - stats["locatable_count"] = len(locatable) - stats["uia_sparse"] = int(stats.get("locatable_count") or 0) < 8 - stats["provider_cold"] = is_provider_cold( - hwnd, findall=int(stats.get("findall_count") or 0) - ) - - # Sparse fallback: light poke + FindAll on a few large same-pid top-levels. - # Includes WeWorkWindow when current target is a thin popup. - # Never merge into SearchResultWindow2 (see allow_same_pid_merge). - stats["merge_same_pid_allowed"] = allow_same_pid_merge - if allow_same_pid_merge and stats["uia_sparse"]: - try: - info = window_info_from_hwnd(hwnd) - merged_n = 0 - # Prefer large main-ish windows first - others: list[tuple[int, int, Any]] = [] - for other in list_same_pid_toplevel_hwnds(info.pid): - if other == hwnd: - continue - rect = _window_rect(other) - if not rect: - continue - area = int(rect.get("width") or 0) * int(rect.get("height") or 0) - if int(rect.get("width") or 0) < 120 or int(rect.get("height") or 0) < 80: - continue - others.append((area, other, rect)) - others.sort(key=lambda x: -x[0]) - for _area, other, _rect in others: - if merged_n >= 8: - break - try: - oinfo = window_info_from_hwnd(other) - except Exception: - continue - if is_inspector_window(oinfo): - continue - if oinfo.class_name in { - "Shell_TrayWnd", - "Shell_SecondaryTrayWnd", - "Progman", - "WorkerW", - "IME", - "MSCTFIME UI", - "tooltips_class32", - }: - continue - # Light poke only — never ShowWindow on siblings - poke_uia_provider(other) - try: - oroot = control_from_hwnd(other) - extra = collect_findall_flat( - oroot, max_nodes=max(100, max_nodes - len(index)) - ) - except Exception: - continue - if len(extra) < 5: - continue - merged_n += 1 - stats.setdefault("merged_same_pid_hwnds", []).append( - { - "hwnd": other, - "title": oinfo.title, - "class_name": oinfo.class_name, - "findall": len(extra), - } - ) - existing = {r.get("path_hint") for r in index} - for row in extra: - row = dict(row) - row["source"] = f"findall_pid:{other}" - if row.get("path_hint") in existing: - continue - index.append(row) - existing.add(row.get("path_hint")) - locatable = locatable_from_index(index) - stats["locatable_count"] = len(locatable) - stats["findall_count"] = max( - int(stats.get("findall_count") or 0), - sum(1 for r in index if str(r.get("source", "")).startswith("findall")), - ) - stats["uia_sparse"] = int(stats.get("locatable_count") or 0) < 8 - except Exception as exc: - stats["same_pid_merge_error"] = str(exc) - - stats["kit_wake"] = kit_report - return tree, index, stats, locatable - - -def collect_findall_flat(root_ctrl, *, max_nodes: int = 8000) -> list[dict[str, Any]]: - """COM FindAll(Subtree) — often finds nodes GetChildren walk misses.""" - out: list[dict[str, Any]] = [] - try: - from uiautomation.uiautomation import Control, _AutomationClient - - ac = _AutomationClient.instance() - cond = ac.IUIAutomation.CreateTrueCondition() - # TreeScope_Subtree = Element|Children|Descendants = 7 - arr = root_ctrl.Element.FindAll(7, cond) - length = int(arr.Length) - for i in range(min(length, max_nodes)): - try: - ele = arr.GetElement(i) - ctrl = Control.CreateControlFromElement(ele) - if ctrl is None: - continue - info = describe_control(ctrl, depth=-1, parent_chain=[]) - out.append( - { - "depth": -1, - "source": "findall", - "name": info.name, - "value": info.value, - "automation_id": info.automation_id, - "control_type": info.control_type, - "class_name": info.class_name, - "is_enabled": info.is_enabled, - "is_offscreen": info.is_offscreen, - "rect": info.rect, - "path_hint": info.path_hint, - } - ) - except Exception: - continue - except Exception: - return out - return out - - -def locatable_from_index( - index: list[dict[str, Any]], *, limit: int = 300 -) -> list[dict[str, Any]]: - """Nodes useful for locator drafting: non-empty Name / Value / AutomationId.""" - rows: list[dict[str, Any]] = [] - seen: set[str] = set() - for r in index: - name = (r.get("name") or "").strip() - value = (r.get("value") or "").strip() - aid = (r.get("automation_id") or "").strip() - if not (name or value or aid): - continue - # Skip ultra-generic noise alone - if name in {"", "pane", "Pane"} and not value and not aid: - continue - key = f"{r.get('control_type')}|{name}|{value}|{aid}" - if key in seen: - continue - seen.add(key) - rows.append(r) - if len(rows) >= limit: - break - # Prefer rows with value or automation_id - rows.sort( - key=lambda x: ( - 0 if (x.get("value") or x.get("automation_id")) else 1, - 0 if x.get("name") else 1, - int(x.get("depth") if x.get("depth") is not None and x.get("depth") >= 0 else 99), - ) - ) - return rows - - -def format_locatable_text(locatable: list[dict[str, Any]], *, limit: int = 120) -> str: - lines = [ - f"可定位候选（Name/Value/AutomationId 非空，共 {len(locatable)}，显示前 {min(limit, len(locatable))}）", - "", - ] - for i, r in enumerate(locatable[:limit]): - lines.append(f"[{i}] {r.get('control_type') or '?'}") - if r.get("name"): - lines.append(f" Name: {r.get('name')}") - if r.get("value"): - lines.append(f" Value: {r.get('value')}") - if r.get("automation_id"): - lines.append(f" AutomationId: {r.get('automation_id')}") - if r.get("class_name"): - lines.append(f" Class: {r.get('class_name')}") - lines.append( - f" offscreen={r.get('is_offscreen')} enabled={r.get('is_enabled')} " - f"src={r.get('source') or 'tree'}" - ) - lines.append("") - if not locatable: - lines.append("（无）UIA 可能过稀；请看截图目录中的 screen.png，或换页后再拾取。") - return "\n".join(lines) - - - -def format_tree_text(tree: dict[str, Any], *, max_lines: int = 2000) -> str: - """Human-readable indented tree for the panel.""" - lines: list[str] = [] - - def walk(node: dict[str, Any], depth: int) -> None: - if len(lines) >= max_lines: - return - indent = " " * depth - ctype = node.get("control_type") or "?" - name = node.get("name") or "" - value = node.get("value") or "" - aid = node.get("automation_id") or "" - bits = [ctype] - if name: - bits.append(f"Name={name!r}") - if value: - bits.append(f"Value={value!r}") - if aid: - bits.append(f"AutomationId={aid!r}") - lines.append(indent + " | ".join(bits)) - for child in node.get("children") or []: - if len(lines) >= max_lines: - lines.append(indent + " …（已截断显示）") - return - walk(child, depth + 1) - - if tree: - walk(tree, 0) - else: - lines.append("(空树)") - if len(lines) >= max_lines: - lines.append(f"… 显示已截断到 {max_lines} 行；完整内容见 tree.json / index.jsonl") - return "\n".join(lines) - - -# Back-compat: older callers expected a single WindowInfo (foreground only). -# Do NOT name this resolve_capture_target — that name is the full 3-tuple API above. -def resolve_frontmost_window( - *, - self_hwnd: int = 0, - poll_sec: float = 0.15, - wait_sec: float = 2.0, -) -> WindowInfo: - _ = poll_sec, wait_sec - info, _cands = find_capture_target( - exclude_hwnds={int(self_hwnd)} if self_hwnd else None, - under_cursor=False, - prefer_wecom_search_popup=False, - ) - return info - - -def hwnd_from_control(ctrl) -> int: - try: - h = int(getattr(ctrl, "NativeWindowHandle", 0) or 0) - if h: - return h - except Exception: - pass - # Walk parents for a native handle - cur = ctrl - for _ in range(24): - try: - h = int(getattr(cur, "NativeWindowHandle", 0) or 0) - if h: - return h - except Exception: - pass - try: - parent = cur.GetParentControl() - except Exception: - break - if parent is None: - break - cur = parent - return 0 - - -def attach_window_by_click(*, timeout_sec: float = 60.0) -> WindowInfo: - """Minimize race: user clicks anywhere on the target window to attach it.""" - x, y = wait_for_left_click(timeout_sec=timeout_sec) - ctrl = control_from_point(x, y) - hwnd = hwnd_from_control(ctrl) - if not hwnd: - # Fallback: window from point via Win32 - hwnd = int(user32.WindowFromPoint(wintypes.POINT(x, y)) or 0) - if not hwnd: - raise RuntimeError(f"点击位置 ({x},{y}) 无法解析到窗口") - info = window_info_from_hwnd(hwnd) - if "UI Inspector" in (info.title or ""): - raise RuntimeError("点到了 Inspector 自己，请点目标软件窗口（如企业微信/添加客户）。") - return info - - -def control_from_hwnd(hwnd: int): - ensure_uia() - import uiautomation as auto - - ctrl = auto.ControlFromHandle(int(hwnd)) - if ctrl is None: - raise RuntimeError(f"无法从 HWND={hwnd} 取得 UIA 根控件") - return ctrl - - -def control_from_point(x: int, y: int): - ensure_uia() - import uiautomation as auto - - ctrl = auto.ControlFromPoint(int(x), int(y)) - if ctrl is None: - raise RuntimeError(f"坐标 ({x},{y}) 处没有 UIA 控件") - return ctrl - - -def build_parent_chain(ctrl, *, max_ancestors: int = 12) -> list[dict[str, str]]: - chain: list[dict[str, str]] = [] - cur = ctrl - for _ in range(max_ancestors): - try: - parent = cur.GetParentControl() - except Exception: - break - if parent is None: - break - chain.append( - { - "name": _safe_str(getattr(parent, "Name", None)), - "value": _read_value(parent), - "automation_id": _safe_str(getattr(parent, "AutomationId", None)), - "control_type": _control_type_name(parent), - "class_name": _safe_str(getattr(parent, "ClassName", None)), - } - ) - cur = parent - chain.reverse() - return chain - - -def dump_tree( - root, - *, - max_depth: int = 16, - max_nodes: int = 8000, - only_visible: bool = False, -) -> tuple[dict[str, Any], list[dict[str, Any]], dict[str, Any]]: - """Return (tree, flat_index, stats). - - Important: even when a node is offscreen / zero-sized, we still recurse into - its children. Pruning the whole subtree was the main cause of empty dumps. - """ - stats = { - "nodes_visited": 0, - "nodes_kept": 0, - "nodes_skipped_self": 0, - "truncated": False, - "max_depth": max_depth, - "max_nodes": max_nodes, - "only_visible": only_visible, - } - flat: list[dict[str, Any]] = [] - - def walk(ctrl, depth: int, ancestors: list[dict[str, str]]) -> Optional[dict[str, Any]]: - if stats["nodes_kept"] >= max_nodes: - stats["truncated"] = True - return None - stats["nodes_visited"] += 1 - - info = describe_control(ctrl, depth=depth, parent_chain=ancestors) - skip_self = False - if only_visible and depth > 0: - rect = info.rect - if info.is_offscreen is True: - skip_self = True - if rect and rect.get("width", 0) <= 0 and rect.get("height", 0) <= 0: - skip_self = True - - child_ancestors = ancestors + [ - { - "name": info.name, - "value": info.value, - "automation_id": info.automation_id, - "control_type": info.control_type, - "class_name": info.class_name, - } - ] - children_out: list[dict[str, Any]] = [] - if depth < max_depth and not stats["truncated"]: - try: - children = ctrl.GetChildren() - except Exception: - children = [] - for child in children or []: - if stats["truncated"] or stats["nodes_kept"] >= max_nodes: - stats["truncated"] = True - break - child_node = walk(child, depth + 1, child_ancestors) - if child_node is not None: - children_out.append(child_node) - - if skip_self: - stats["nodes_skipped_self"] += 1 - # Pass children up so hierarchy is not lost - if len(children_out) == 1: - return children_out[0] - if children_out: - return { - "control_type": info.control_type or "SkippedContainer", - "name": info.name, - "value": info.value, - "automation_id": info.automation_id, - "class_name": info.class_name, - "skipped_self": True, - "children": children_out, - } - return None - - stats["nodes_kept"] += 1 - node = info.to_dict() - flat.append( - { - "depth": depth, - "source": "tree", - "name": info.name, - "value": info.value, - "automation_id": info.automation_id, - "control_type": info.control_type, - "class_name": info.class_name, - "is_enabled": info.is_enabled, - "is_offscreen": info.is_offscreen, - "rect": info.rect, - "path_hint": info.path_hint, - } - ) - if children_out: - node["children"] = children_out - return node - - tree = walk(root, 0, []) or {} - return tree, flat, stats - - -def wait_for_left_click( - *, - timeout_sec: float = 60.0, - poll_sec: float = 0.05, -) -> tuple[int, int]: - """Block until left mouse button is pressed then released; return cursor pos.""" - VK_LBUTTON = 0x01 - - def down() -> bool: - return bool(user32.GetAsyncKeyState(VK_LBUTTON) & 0x8000) - - deadline = time.time() + timeout_sec - # Wait until button is up first (avoid consuming the GUI click that started pick) - while time.time() < deadline and down(): - time.sleep(poll_sec) - - while time.time() < deadline: - if down(): - while time.time() < deadline and down(): - time.sleep(poll_sec) - point = wintypes.POINT() - user32.GetCursorPos(ctypes.byref(point)) - return int(point.x), int(point.y) - time.sleep(poll_sec) - raise TimeoutError(f"等待鼠标左键点击超时（{timeout_sec:.0f}s）") - - -def pick_control_at_cursor_after_click( - *, - timeout_sec: float = 60.0, - attached_hwnd: Optional[int] = None, -) -> tuple[ControlInfo, dict[str, int]]: - x, y = wait_for_left_click(timeout_sec=timeout_sec) - ctrl = control_from_point(x, y) - info_hwnd = hwnd_from_control(ctrl) or int( - user32.WindowFromPoint(wintypes.POINT(int(x), int(y))) or 0 - ) - top = get_toplevel_hwnd(info_hwnd) if info_hwnd else 0 - - # Taskbar / desktop chrome — almost never what you want for skill RPA - class_buf = ctypes.create_unicode_buffer(256) - if top: - user32.GetClassNameW(top, class_buf, 256) - top_class = class_buf.value or "" - if top_class in {"Shell_TrayWnd", "Shell_SecondaryTrayWnd", "NotifyIconOverflowWindow"}: - raise RuntimeError( - f"点到了任务栏/系统托盘（class={top_class}），不是业务窗口。\n" - f"请点已附着窗口内部的控件，不要点任务栏上的 Python/Cursor 图标。" - ) - - if attached_hwnd: - attached_top = get_toplevel_hwnd(int(attached_hwnd)) - if top and attached_top and top != attached_top: - title_buf = ctypes.create_unicode_buffer(512) - user32.GetWindowTextW(top, title_buf, 512) - raise RuntimeError( - "拾取点不在已附着窗口内。\n" - f"已附着 hwnd={attached_top}；实际点到 title={title_buf.value!r} " - f"class={top_class!r} hwnd={top}。\n" - "请重新「点选附着」到目标软件（如企业微信），再在该窗口内拾取。" - ) - - chain = build_parent_chain(ctrl) - info = describe_control(ctrl, depth=0, parent_chain=chain) - return info, {"x": x, "y": y} diff --git a/tools/ui_inspector/diagnostics/__init__.py b/tools/ui_inspector/diagnostics/__init__.py deleted file mode 100644 index d70cdb9..0000000 --- a/tools/ui_inspector/diagnostics/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Capture validation and timeline diagnostics.""" - -from .capture_validator import validate_capture -from .replay_preflight import build_replay_preflight, write_replay_preflight - -__all__ = ["validate_capture", "build_replay_preflight", "write_replay_preflight"] diff --git a/tools/ui_inspector/diagnostics/capture_validator.py b/tools/ui_inspector/diagnostics/capture_validator.py deleted file mode 100644 index d210572..0000000 --- a/tools/ui_inspector/diagnostics/capture_validator.py +++ /dev/null @@ -1,134 +0,0 @@ -"""Product-neutral capture quality checks.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - - -def _read(path: Path) -> dict[str, Any]: - return json.loads(path.read_text(encoding="utf-8")) - - -def validate_capture(flow_dir: Path) -> dict[str, Any]: - root = Path(flow_dir) - issues: list[dict[str, Any]] = [] - actions = sorted( - (_read(path) for path in (root / "actions").glob("*.json")), - key=lambda item: int(item.get("sequence") or 0), - ) - expected = list(range(1, len(actions) + 1)) - actual = [int(item.get("sequence") or 0) for item in actions] - if actual != expected: - issues.append({"severity": "error", "code": "ACTION_SEQUENCE_GAP", "actual": actual}) - elements = { - item.get("id"): item - for item in (_read(path) for path in (root / "elements").glob("*.json")) - if item.get("id") - } - interactive_types = {"click", "fill", "type", "press_key", "select", "toggle"} - previous_action: dict[str, Any] | None = None - for action in actions: - target = dict(action.get("target_snapshot") or {}) - rect = dict(target.get("rect") or target.get("bounds") or {}) - area = float(rect.get("width") or 0) * float(rect.get("height") or 0) - if action.get("action_type") in {"click", "fill", "select", "toggle"} and not target: - issues.append({"severity": "error", "code": "ACTION_TARGET_MISSING", "action_id": action.get("id")}) - if action.get("action_type") == "click" and area > 1_000_000: - issues.append({"severity": "error", "code": "CLICK_TARGET_OVERSIZED", "action_id": action.get("id"), "area": area}) - target_name = str(target.get("name") or target.get("automation_id") or "").lower() - if ( - action.get("surface") == "windows" - and action.get("action_type") == "click" - and str(target.get("control_type") or "") in {"TreeControl", "PaneControl", "GroupControl"} - and any(token in target_name for token in ("search_result", "listview", "list_view", "watermark", "container", "layout")) - ): - issues.append({"severity": "warning", "code": "CLICK_TARGET_GENERIC_CONTAINER", "action_id": action.get("id"), "target_name": target_name}) - if ( - action.get("surface") == "windows" - and action.get("action_type") == "press_key" - and (str(target.get("control_type") or "") == "WindowControl" or area > 1_000_000) - ): - issues.append({"severity": "warning", "code": "KEY_TARGET_ROOT_WINDOW", "action_id": action.get("id")}) - if action.get("action_type") == "click" and str(target.get("tag") or "").lower() in {"path", "use", "g", "svg"}: - issues.append({"severity": "warning", "code": "GRAPHIC_LEAF_TARGET", "action_id": action.get("id"), "tag": target.get("tag")}) - evidence = dict(action.get("evidence") or {}) - if not action.get("after_observation_id") and not evidence.get("eventual_observation_id"): - issues.append({"severity": "warning", "code": "ACTION_EVIDENCE_MISSING", "action_id": action.get("id")}) - if evidence.get("locator_unique") is False: - issues.append({"severity": "warning", "code": "LOCATOR_NOT_UNIQUE", "action_id": action.get("id")}) - if action.get("surface") == "browser" and action.get("action_type") in interactive_types: - element_id = action.get("target_element_id") - if target and not element_id: - issues.append({"severity": "warning", "code": "TARGET_ELEMENT_REFERENCE_MISSING", "action_id": action.get("id")}) - element = elements.get(element_id) if element_id else None - primary = dict((element or {}).get("primary_locator") or {}) - if element_id and (not primary or not primary.get("production_eligible")): - issues.append({"severity": "warning", "code": "ACTION_PRODUCTION_LOCATOR_MISSING", "action_id": action.get("id"), "element_id": element_id}) - elif primary and float(primary.get("stability_score") or 0.0) < 0.55: - issues.append({"severity": "warning", "code": "LOW_STABILITY_LOCATOR", "action_id": action.get("id"), "element_id": element_id, "strategy": primary.get("strategy"), "stability_score": primary.get("stability_score")}) - raw_meta = dict((action.get("raw_event") or {}).get("meta") or {}) - if not raw_meta.get("event_id"): - issues.append({"severity": "warning", "code": "ACTION_RAW_EVENT_LINK_MISSING", "action_id": action.get("id")}) - if action.get("surface") == "windows" and action.get("action_type") in interactive_types: - element_id = action.get("target_element_id") - if target and not element_id: - issues.append({"severity": "warning", "code": "TARGET_ELEMENT_REFERENCE_MISSING", "action_id": action.get("id")}) - element = elements.get(element_id) if element_id else None - primary = dict((element or {}).get("primary_locator") or {}) - if element_id and not primary: - issues.append({"severity": "warning", "code": "ACTION_PRODUCTION_LOCATOR_MISSING", "action_id": action.get("id"), "element_id": element_id}) - if previous_action and action.get("action_type") == "fill" and previous_action.get("action_type") == "fill": - same_target = bool(action.get("target_element_id")) and action.get("target_element_id") == previous_action.get("target_element_id") - old_text = str((previous_action.get("input") or {}).get("text") or "") - new_text = str((action.get("input") or {}).get("text") or "") - old_ms = ((previous_action.get("raw_event") or {}).get("meta") or {}).get("source_ts_ms") - new_ms = ((action.get("raw_event") or {}).get("meta") or {}).get("source_ts_ms") - close = False - try: - close = 0 <= float(new_ms) - float(old_ms) <= 2000 - except (TypeError, ValueError): - pass - if same_target and old_text and new_text.startswith(old_text) and new_text != old_text and close: - issues.append({"severity": "warning", "code": "DUPLICATE_FILL_TRANSACTION", "action_id": action.get("id"), "previous_action_id": previous_action.get("id"), "element_id": action.get("target_element_id")}) - previous_action = action - journal = root / "journal" / "raw_events.jsonl" - if not journal.exists(): - issues.append({"severity": "error", "code": "RAW_EVENT_JOURNAL_MISSING"}) - if not actions: - issues.append({"severity": "error", "code": "ZERO_ACTIONS"}) - typing_transactions = 0 - if journal.exists(): - for line in journal.read_text(encoding="utf-8").splitlines(): - try: - payload = json.loads(line) - except (TypeError, ValueError, json.JSONDecodeError): - continue - event = dict(payload.get("event") or payload) - if event.get("kind") == "typing_activity": - typing_transactions += 1 - fill_count = sum( - 1 for action in actions if action.get("action_type") in {"fill", "type"} - ) - if typing_transactions > fill_count: - issues.append( - { - "severity": "warning", - "code": "TEXT_INPUT_TRANSACTION_MISSING", - "typing_transactions": typing_transactions, - "fill_actions": fill_count, - } - ) - screenshot_count = len(list((root / "observations").glob("*/screen.png"))) - return { - "schema": "openclaw.ui_inspector.validation.v1", - "flow_dir": str(root), - "action_count": len(actions), - "element_count": len(elements), - "observation_count": len(list((root / "observations").glob("*/meta.json"))), - "screenshot_count": screenshot_count, - "error_count": sum(1 for issue in issues if issue["severity"] == "error"), - "warning_count": sum(1 for issue in issues if issue["severity"] == "warning"), - "issues": issues, - } diff --git a/tools/ui_inspector/diagnostics/replay_preflight.py b/tools/ui_inspector/diagnostics/replay_preflight.py deleted file mode 100644 index 3e5ca40..0000000 --- a/tools/ui_inspector/diagnostics/replay_preflight.py +++ /dev/null @@ -1,133 +0,0 @@ -"""Static, non-mutating replay readiness report for a finished capture.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - - -_INTERACTIVE = {"click", "fill", "type", "press_key", "select", "toggle"} - - -def _read(path: Path) -> dict[str, Any]: - return json.loads(path.read_text(encoding="utf-8")) - - -def build_replay_preflight(flow_dir: Path) -> dict[str, Any]: - """Assess whether recorded actions contain executable, verified locators. - - This deliberately does not click or navigate. `event_time_unique` means - the locator count was measured in the event-origin document during capture; - it is not represented as a cross-session replay result. - """ - root = Path(flow_dir) - actions = sorted( - (_read(path) for path in (root / "actions").glob("*.json")), - key=lambda item: int(item.get("sequence") or 0), - ) - elements = { - item.get("id"): item - for item in (_read(path) for path in (root / "elements").glob("*.json")) - if item.get("id") - } - rows: list[dict[str, Any]] = [] - for action in actions: - action_type = str(action.get("action_type") or "") - if action_type not in _INTERACTIVE: - rows.append({ - "action_id": action.get("id"), - "sequence": action.get("sequence"), - "action_type": action_type, - "result": "context_only", - "reason": "non_interactive_or_navigation", - }) - continue - element_id = action.get("target_element_id") - element = elements.get(element_id) or {} - primary = dict(element.get("primary_locator") or {}) - eligible = bool(primary.get("production_eligible")) - unique = primary.get("unique") is True - score = float(primary.get("stability_score") or 0.0) - result = "pass" if eligible and unique and score >= 0.55 else "warning" - if not eligible or not unique: - result = "fail" - target = dict(action.get("target_snapshot") or {}) - bounds = dict(target.get("bounds") or target.get("rect") or {}) - area = float(bounds.get("width") or 0) * float(bounds.get("height") or 0) - oversized_click = action_type == "click" and area > 1_000_000 - root_window_target = ( - action_type == "click" - and str(target.get("control_type") or "") == "WindowControl" - ) - if oversized_click or root_window_target: - result = "fail" - validation = str(primary.get("validation") or "none") - verification = ( - "event_time_unique" if validation == "live" and unique - else "snapshot_unique" if validation in {"snapshot_only", "live_tree"} and unique - else "not_validated" - ) - raw_meta = dict((action.get("raw_event") or {}).get("meta") or {}) - rows.append({ - "action_id": action.get("id"), - "sequence": action.get("sequence"), - "action_type": action_type, - "result": result, - "element_id": element_id, - "logical_name": element.get("logical_name") or "", - "locator": primary.get("expression") or "", - "strategy": primary.get("strategy") or "", - "stability_score": score, - "match_count": primary.get("actual_count"), - "verification": verification, - "page_id": raw_meta.get("page_id") or "", - "document_id": raw_meta.get("document_id") or "", - "raw_event_ids": list(action.get("raw_event_ids") or []), - "reason": ( - "oversized_or_root_window_click_target" - if oversized_click or root_window_target else "" - ), - }) - counts = { - name: sum(1 for row in rows if row.get("result") == name) - for name in ("pass", "warning", "fail", "context_only") - } - from diagnostics.capture_validator import validate_capture - - quality = validate_capture(root) - blocking_codes = { - "TEXT_INPUT_TRANSACTION_MISSING", - "CLICK_TARGET_GENERIC_CONTAINER", - "KEY_TARGET_ROOT_WINDOW", - "CLICK_TARGET_OVERSIZED", - "ACTION_TARGET_MISSING", - "ZERO_ACTIONS", - } - quality_blockers = [ - issue - for issue in quality.get("issues") or [] - if issue.get("severity") == "error" or issue.get("code") in blocking_codes - ] - counts["quality_blockers"] = len(quality_blockers) - return { - "schema": "openclaw.ui_inspector.replay_preflight.v1", - "mode": "static_non_mutating", - "note": "事件时刻唯一性不等同于跨会话真实回放；本报告不会点击、输入或导航。", - "flow_dir": str(root), - "ready": not quality_blockers and counts.get("fail", 0) == 0, - "summary": counts, - "quality_gate": { - "passed": not quality_blockers, - "issues": quality_blockers, - }, - "actions": rows, - } - - -def write_replay_preflight(flow_dir: Path) -> dict[str, Any]: - report = build_replay_preflight(flow_dir) - path = Path(flow_dir) / "diagnostics" / "replay_report.json" - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - return report diff --git a/tools/ui_inspector/drivers/__init__.py b/tools/ui_inspector/drivers/__init__.py deleted file mode 100644 index c1f7697..0000000 --- a/tools/ui_inspector/drivers/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Surface drivers for UI Inspector (windows / browser / mobile stub).""" - -from __future__ import annotations - -from .base import DumpResult, SurfaceDriver -from .browser_pw import BrowserPlaywrightDriver -from .mobile import MobileDriver -from .windows_uia import WindowsUiaDriver -import target as _target - - -def get_driver(surface: str) -> SurfaceDriver: - if surface == _target.SURFACE_WINDOWS: - return WindowsUiaDriver() - if surface == _target.SURFACE_BROWSER: - return BrowserPlaywrightDriver() - if surface in {_target.SURFACE_ANDROID, _target.SURFACE_IOS}: - return MobileDriver(surface=surface) - raise RuntimeError(f"ERROR:SURFACE_UNSUPPORTED unknown surface={surface!r}") - - -__all__ = [ - "DumpResult", - "SurfaceDriver", - "WindowsUiaDriver", - "BrowserPlaywrightDriver", - "MobileDriver", - "get_driver", -] diff --git a/tools/ui_inspector/drivers/base.py b/tools/ui_inspector/drivers/base.py deleted file mode 100644 index 6898720..0000000 --- a/tools/ui_inspector/drivers/base.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Driver protocol: dump one step's UI tree for a surface.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Any, Optional, Protocol - - -@dataclass -class DumpResult: - surface: str - driver: str - window: dict[str, Any] - tree: dict[str, Any] - index: list[dict[str, Any]] - locatable: list[dict[str, Any]] - stats: dict[str, Any] - selection: dict[str, Any] = field(default_factory=dict) - url: str = "" - page_title: str = "" - locatable_text: str = "" - tree_text: str = "" - screenshot_bytes: Optional[bytes] = None # if set, artifacts write screen.png from bytes - use_hwnd_screenshot: bool = True # windows: ImageGrab by hwnd - - -class SurfaceDriver(Protocol): - def dump( - self, - *, - info: Any, - selection_meta: dict[str, Any], - max_depth: int = 16, - max_nodes: int = 8000, - warmup_mode: str = "full", - merge_same_pid_if_sparse: bool = True, - warmed_pids: Optional[set[int]] = None, - ) -> DumpResult: ... diff --git a/tools/ui_inspector/drivers/browser_pw.py b/tools/ui_inspector/drivers/browser_pw.py deleted file mode 100644 index 2f677bb..0000000 --- a/tools/ui_inspector/drivers/browser_pw.py +++ /dev/null @@ -1,90 +0,0 @@ -"""Browser whole-page dump via Inspector Playwright session + kit page_dump.""" - -from __future__ import annotations - -from typing import Any, Optional - -import kit_browser -import target -from .base import DumpResult - - -class BrowserPlaywrightDriver: - surface = target.SURFACE_BROWSER - driver_name = "playwright" - - def dump( - self, - *, - info: Any, - selection_meta: dict[str, Any], - max_depth: int = 16, - max_nodes: int = 8000, - warmup_mode: str = "full", - merge_same_pid_if_sparse: bool = True, - warmed_pids: Optional[set[int]] = None, - page_id: str = "", - document_id: str = "", - frame_id: str = "", - ) -> DumpResult: - del max_depth, warmup_mode, merge_same_pid_if_sparse, warmed_pids # unused - sess = kit_browser.ensure_session() - dump = kit_browser.dump_current_page( - max_nodes=max_nodes, locatable_limit=300, page_id=page_id - ) - try: - png = kit_browser.screenshot_png(page_id=page_id) - except Exception: - png = None - - from jiangchang_skill_core.rpa.page_dump import ( - format_locatable_text, - format_tree_text, - ) - - url = dump.get("url") or "" - page_title = dump.get("title") or "" - window = info.to_dict() if hasattr(info, "to_dict") else dict(info or {}) - # Prefer live page identity for labeling / meta - window = { - **window, - "title": page_title or window.get("title") or "", - "class_name": window.get("class_name") or "PlaywrightPage", - "process_name": window.get("process_name") or "chrome.exe", - "url": url, - } - stats = dict(dump.get("stats") or {}) - active_page = dict(stats.get("active_page") or {}) - stats["browser_session"] = { - "profile_dir": sess.get("profile_dir"), - "reused": sess.get("reused"), - "page_count": active_page.get("page_count") or sess.get("page_count"), - "resolve": active_page.get("resolve") or "visible", - "note": ( - "dump 来自 Inspector 自管 Playwright 的当前可见标签页" - "（document.visibilityState），不是系统 Chrome 的 UIA，" - "也不是固定的 pages[0]。多标签时请切到目标页再拾取。" - ), - } - sel = dict(selection_meta or {}) - sel["surface"] = self.surface - sel["browser_url"] = url - sel["page_id"] = page_id or active_page.get("page_id") or "" - sel["document_id"] = document_id - sel["frame_id"] = frame_id - return DumpResult( - surface=self.surface, - driver=self.driver_name, - window=window, - tree=dump.get("tree") or {}, - index=list(dump.get("index") or []), - locatable=list(dump.get("locatable") or []), - stats=stats, - selection=sel, - url=url, - page_title=page_title, - locatable_text=format_locatable_text(dump.get("locatable") or []), - tree_text=format_tree_text(dump.get("tree") or {}), - screenshot_bytes=png, - use_hwnd_screenshot=False, - ) diff --git a/tools/ui_inspector/drivers/mobile.py b/tools/ui_inspector/drivers/mobile.py deleted file mode 100644 index 10145c0..0000000 --- a/tools/ui_inspector/drivers/mobile.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Mobile surface stub — reserved for future android/ios dump.""" - -from __future__ import annotations - -from typing import Any, Optional - -import target -from .base import DumpResult - - -class MobileDriver: - def __init__(self, *, surface: str = target.SURFACE_ANDROID) -> None: - self.surface = surface - self.driver_name = "mobile_stub" - - def dump( - self, - *, - info: Any, - selection_meta: dict[str, Any], - max_depth: int = 16, - max_nodes: int = 8000, - warmup_mode: str = "full", - merge_same_pid_if_sparse: bool = True, - warmed_pids: Optional[set[int]] = None, - ) -> DumpResult: - raise RuntimeError( - f"ERROR:SURFACE_UNSUPPORTED 手机端拾取尚未实现（surface={self.surface}）。" - "请先用桌面或浏览器目标。" - ) diff --git a/tools/ui_inspector/drivers/windows_uia.py b/tools/ui_inspector/drivers/windows_uia.py deleted file mode 100644 index f27f5f0..0000000 --- a/tools/ui_inspector/drivers/windows_uia.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Windows desktop dump via existing desktop_uia.""" - -from __future__ import annotations - -from typing import Any, Optional - -import desktop_uia as uia -import target -from .base import DumpResult - - -class WindowsUiaDriver: - surface = target.SURFACE_WINDOWS - driver_name = "uiautomation" - - def dump( - self, - *, - info: Any, - selection_meta: dict[str, Any], - max_depth: int = 16, - max_nodes: int = 8000, - warmup_mode: str = "full", - merge_same_pid_if_sparse: bool = True, - warmed_pids: Optional[set[object]] = None, - ) -> DumpResult: - warmed = warmed_pids if warmed_pids is not None else set() - mode = warmup_mode - warm_key = (int(info.pid), str(info.class_name or "")) - if (warm_key in warmed or int(info.pid) in warmed) and mode == "full": - mode = "light" - merge_sparse = merge_same_pid_if_sparse - if (info.class_name or "") == "SearchResultWindow2": - merge_sparse = False - tree, index, stats, locatable = uia.dump_window_tree( - info.hwnd, - max_depth=max_depth, - max_nodes=max_nodes, - only_visible=False, - warmup_mode=mode, - merge_same_pid_if_sparse=merge_sparse, - ) - return DumpResult( - surface=self.surface, - driver=self.driver_name, - window=info.to_dict(), - tree=tree, - index=index, - locatable=locatable, - stats=stats, - selection=dict(selection_meta or {}), - url="", - page_title="", - locatable_text=uia.format_locatable_text(locatable), - tree_text=uia.format_tree_text(tree), - screenshot_bytes=None, - use_hwnd_screenshot=True, - ) diff --git a/tools/ui_inspector/evidence/__init__.py b/tools/ui_inspector/evidence/__init__.py deleted file mode 100644 index ca5e14b..0000000 --- a/tools/ui_inspector/evidence/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Asynchronous evidence capture.""" - -from .coordinator import EvidenceCoordinator - -__all__ = ["EvidenceCoordinator"] - diff --git a/tools/ui_inspector/evidence/coordinator.py b/tools/ui_inspector/evidence/coordinator.py deleted file mode 100644 index 7800023..0000000 --- a/tools/ui_inspector/evidence/coordinator.py +++ /dev/null @@ -1,255 +0,0 @@ -"""Coalesced background full-page evidence capture. - -Semantic actions are committed immediately. Slow dumps are supplementary -evidence and are explicitly labelled eventual instead of being misrepresented -as an event-time after-state. -""" - -from __future__ import annotations - -import queue -import threading -import time -from dataclasses import dataclass -from typing import Any, Callable, Optional - - -@dataclass -class EvidenceRequest: - action_id: str - context: dict[str, Any] - requested_monotonic: float - - -class EvidenceCoordinator: - def __init__( - self, - manager: Any, - dump_fn: Callable[..., Any], - *, - coalesce_sec: float = 0.25, - on_dump: Optional[Callable[[Any], None]] = None, - ) -> None: - self.manager = manager - self.dump_fn = dump_fn - self.coalesce_sec = coalesce_sec - self.on_dump = on_dump - self._queue: queue.Queue[EvidenceRequest | None] = queue.Queue(maxsize=512) - self._urgent_queue: queue.Queue[EvidenceRequest | None] = queue.Queue(maxsize=128) - self._thread: Optional[threading.Thread] = None - self._urgent_thread: Optional[threading.Thread] = None - self._stop = threading.Event() - self.dropped = 0 - self.capture_count = 0 - self.failure_count = 0 - - def start(self) -> None: - if self._thread and self._thread.is_alive(): - return - self._stop.clear() - self._thread = threading.Thread(target=self._run, name="ui-inspector-evidence", daemon=True) - self._thread.start() - if str(getattr(self.manager.action_flow, "surface", "")) == "windows": - self._urgent_thread = threading.Thread( - target=self._run_urgent, - name="ui-inspector-evidence-transient", - daemon=True, - ) - self._urgent_thread.start() - - def schedule(self, action: Any) -> None: - raw = dict(getattr(action, "raw_event", {}) or {}) - meta = dict(raw.get("meta") or {}) - context = { - "page_id": meta.get("page_id") or "", - "document_id": meta.get("document_id") or "", - "frame_id": meta.get("frame_id") or "", - "source_ts_ms": meta.get("source_ts_ms"), - "url": raw.get("url") or "", - } - scope = dict((raw.get("target") or {}).get("window_scope") or {}) - context.update( - { - "target_hwnd": int(scope.get("hwnd") or 0), - "target_pid": int(scope.get("pid") or 0), - "target_class_name": str(scope.get("class_name") or ""), - "target_title": str(scope.get("title") or ""), - } - ) - out_queue = ( - self._urgent_queue - if context["target_class_name"] == "SearchResultWindow2" - else self._queue - ) - try: - out_queue.put_nowait(EvidenceRequest(action.id, context, time.monotonic())) - except queue.Full: - self.dropped += 1 - - def prime(self) -> None: - """Start one background enrichment without creating a semantic action.""" - try: - self._queue.put_nowait(EvidenceRequest("", {}, time.monotonic())) - except queue.Full: - self.dropped += 1 - - def _dump(self, context: dict[str, Any]) -> Any: - kwargs = { - k: context.get(k) - for k in ( - "page_id", "document_id", "frame_id", "target_hwnd", - "target_pid", "target_class_name", "target_title", - ) - if context.get(k) - } - if kwargs: - try: - return self.dump_fn(**kwargs) - except TypeError as exc: - if "unexpected keyword" not in str(exc) and "positional" not in str(exc): - raise - return self.dump_fn() - - def _run(self) -> None: - while not self._stop.is_set(): - try: - first = self._queue.get(timeout=0.2) - except queue.Empty: - continue - if first is None: - self._queue.task_done() - self._stop.set() - break - batch = [first] - stop_after_batch = False - deadline = time.monotonic() + self.coalesce_sec - while time.monotonic() < deadline: - try: - item = self._queue.get(timeout=max(0.01, deadline - time.monotonic())) - except queue.Empty: - break - if item is None: - self._queue.task_done() - stop_after_batch = True - break - batch.append(item) - - groups: dict[str, list[EvidenceRequest]] = {} - for item in batch: - key = str(item.context.get("page_id") or "active") - groups.setdefault(key, []).append(item) - for items in groups.values(): - latest = items[-1] - try: - dump = self._dump(latest.context) - if self.on_dump is not None: - self.on_dump(dump) - self.capture_count += 1 - # A desktop provider dump can take seconds. Actions that - # arrived while it was running are still earlier than this - # eventual observation, so fold them into the same result - # instead of immediately launching another identical dump. - if str(getattr(self.manager.action_flow, "surface", "")) == "windows": - while True: - try: - late = self._queue.get_nowait() - except queue.Empty: - break - if late is None: - self._queue.task_done() - stop_after_batch = True - break - batch.append(late) - items.append(late) - if not self._stop.is_set(): - self.manager.attach_eventual_evidence( - [item.action_id for item in items], dump=dump, - requested_monotonic=min(item.requested_monotonic for item in items), - ) - except Exception as exc: - self.failure_count += 1 - if not self._stop.is_set(): - self.manager.note_evidence_failure([item.action_id for item in items], str(exc)) - for _item in batch: - self._queue.task_done() - if stop_after_batch: - self._stop.set() - - def _run_urgent(self) -> None: - """Capture transient native popups independently of a slow main dump.""" - while not self._stop.is_set(): - try: - first = self._urgent_queue.get(timeout=0.1) - except queue.Empty: - continue - if first is None: - self._urgent_queue.task_done() - break - batch = [first] - # A very short coalesce keeps double-clicks together without waiting - # long enough for a search-result popup to disappear. - deadline = time.monotonic() + 0.04 - while time.monotonic() < deadline: - try: - item = self._urgent_queue.get_nowait() - except queue.Empty: - time.sleep(0.005) - continue - if item is None: - self._urgent_queue.task_done() - break - if item.context.get("target_hwnd") == first.context.get("target_hwnd"): - batch.append(item) - else: - # Preserve a different transient window for the next pass. - try: - self._urgent_queue.put_nowait(item) - except queue.Full: - self.dropped += 1 - self._urgent_queue.task_done() - break - try: - dump = self._dump(batch[-1].context) - if self.on_dump is not None: - self.on_dump(dump) - self.capture_count += 1 - if not self._stop.is_set(): - self.manager.attach_eventual_evidence( - [item.action_id for item in batch], - dump=dump, - requested_monotonic=min(item.requested_monotonic for item in batch), - ) - except Exception as exc: - self.failure_count += 1 - if not self._stop.is_set(): - self.manager.note_evidence_failure( - [item.action_id for item in batch], str(exc) - ) - finally: - for _item in batch: - self._urgent_queue.task_done() - - def stop(self, *, timeout: float = 35.0) -> None: - if not self._thread: - return - # Drain the transient-window lane first. A WeCom main-window dump can - # be stuck in an uninterruptible provider call for tens of seconds; it - # must not cancel the short-lived popup evidence when the user ends the - # flow immediately after selecting a result. - if self._urgent_thread: - try: - self._urgent_queue.put(None, timeout=1.0) - except queue.Full: - pass - self._urgent_thread.join(timeout=min(max(timeout, 3.0), 8.0)) - try: - self._queue.put(None, timeout=2.0) - except queue.Full: - pass - self._thread.join(timeout=timeout) - if self._thread.is_alive(): - # A provider dump may be uninterruptible. Prevent a late result - # from mutating a manager which is already being finalized. - self._stop.set() - self._thread = None - self._urgent_thread = None diff --git a/tools/ui_inspector/flow_session.py b/tools/ui_inspector/flow_session.py deleted file mode 100644 index c37766d..0000000 --- a/tools/ui_inspector/flow_session.py +++ /dev/null @@ -1,408 +0,0 @@ -"""Flow session glue: v2 ActionRecordingManager + surface recorders + v1 dual-write.""" - -from __future__ import annotations - -import queue -import threading -import uuid -from pathlib import Path -from typing import Any, Callable, Optional - -import artifacts as art -from core.pipeline.event_journal import EventJournal -from evidence.coordinator import EvidenceCoordinator -from recorder.base import RawUserEvent -from recorder.manager import ActionRecordingManager - - -class FlowSession: - def __init__( - self, - flow_dir: Path, - *, - flow_name: str, - surface: str, - recording_mode: str = "auto", - dump_fn: Optional[Callable[[], Any]] = None, - write_v1_step: Optional[Callable[..., Any]] = None, - exclude_hwnds: Optional[Callable[[], set[int]]] = None, - on_action_update: Optional[Callable[[Any, str], None]] = None, - stability_timeout_sec: float = 3.0, - ) -> None: - self.flow_dir = Path(flow_dir) - self.surface = surface - self.recording_mode = recording_mode - self._dump_fn = dump_fn - self.exclude_hwnds = exclude_hwnds or (lambda: set()) - self.manager = ActionRecordingManager( - self.flow_dir, - flow_name=flow_name, - flow_id=self.flow_dir.name, - surface=surface, - recording_mode=recording_mode, - dump_fn=dump_fn, - write_v1_step=write_v1_step, - stability_timeout_sec=stability_timeout_sec, - defer_action_evidence=( - surface in {"browser", "windows"} - and recording_mode != "legacy_interval" - ), - on_action_update=on_action_update, - ) - self.journal = EventJournal(self.flow_dir) - self._evidence = ( - EvidenceCoordinator(self.manager, dump_fn, on_dump=self._on_evidence_dump) - if surface in {"browser", "windows"} - and dump_fn is not None - and recording_mode != "legacy_interval" - else None - ) - if self._evidence is not None: - self.manager.on_action = self._evidence.schedule - self._recorder = None - self._recorder_health: dict[str, Any] = {} - self._event_queue: queue.Queue[tuple[str, Any]] = queue.Queue(maxsize=512) - self._event_thread: Optional[threading.Thread] = None - self._event_stop = threading.Event() - self._stats_lock = threading.Lock() - self._stats_cache: dict[str, Any] = { - "recorder_status": "idle", - "surface_recorder": "off", - "state_count": 0, - "observation_count": 0, - "action_count": 0, - "element_count": 0, - "queued_events": 0, - "dropped_events": 0, - } - self._dropped_events = 0 - - def start(self, *, entry_dump: Optional[Any] = None) -> None: - if self.recording_mode == "legacy_interval": - self.manager.start(entry_dump=entry_dump) - self._refresh_stats() - return - # Start observation before entry persistence. Events are journaled and - # buffered until manager.start() completes, eliminating startup loss. - self._start_surface_recorder() - if self.surface == "windows" and entry_dump is not None: - self._on_evidence_dump(entry_dump) - if self._evidence is not None: - self._evidence.start() - try: - self.manager.start(entry_dump=entry_dump) - except Exception: - if self._recorder: - self._recorder.stop() - self._recorder = None - if self._evidence is not None: - self._evidence.stop(timeout=3.0) - raise - self._start_event_consumer() - if self.surface == "windows" and self._evidence is not None: - # Warm/enrich UIA immediately, but on the evidence worker. Raw - # recording and Action reduction remain fully independent. - self._evidence.prime() - self._refresh_stats() - - def _submit(self, kind: str, payload: Any = None) -> bool: - try: - if kind == "stop": - self._event_queue.put((kind, payload), timeout=2.0) - else: - self._event_queue.put_nowait((kind, payload)) - return True - except queue.Full: - self._dropped_events += 1 - self.manager.last_warning = ( - f"recorder queue full; dropped={self._dropped_events}" - ) - self._refresh_stats() - return False - - def _start_event_consumer(self) -> None: - if self._event_thread and self._event_thread.is_alive(): - return - self._event_stop.clear() - - def consume() -> None: - while not self._event_stop.is_set(): - try: - kind, payload = self._event_queue.get(timeout=0.2) - except queue.Empty: - continue - try: - if kind == "stop": - break - if kind == "event": - self.manager.handle_raw_event(payload) - elif kind == "heartbeat": - self.manager.heartbeat() - elif kind == "pause": - self.manager.pause() - elif kind == "resume": - self.manager.resume() - except Exception as exc: - self.manager.last_warning = str(exc) - finally: - self._event_queue.task_done() - self._refresh_stats() - self._event_stop.set() - - self._event_thread = threading.Thread( - target=consume, - name="ui-inspector-action-consumer", - daemon=True, - ) - self._event_thread.start() - - def _start_surface_recorder(self) -> None: - def on_event(ev: RawUserEvent) -> None: - # Preserve the observed order and bound memory/thread growth. - if self.surface == "windows" and not (ev.meta or {}).get("event_id"): - ev.meta["event_id"] = f"windows:{uuid.uuid4().hex}" - self.journal.append(ev, channel="semantic") - self._submit("event", ev) - - def on_raw_event(payload: dict[str, Any]) -> None: - self.journal.append(payload, channel="browser_dom") - - if self.surface == "browser": - from surfaces.browser import BrowserRecorder - - self._recorder = BrowserRecorder( - on_event=on_event, on_raw_event=on_raw_event - ) - else: - from surfaces.windows import WindowsRecorder - - self._recorder = WindowsRecorder( - on_event=on_event, - exclude_hwnds=self.exclude_hwnds, - ) - self._seed_windows_element_atlas() - self._recorder.start() - - def _on_evidence_dump(self, dump: Any) -> None: - recorder = self._recorder - if self.surface != "windows" or recorder is None or not hasattr(recorder, "learn_elements"): - return - try: - recorder.learn_elements( - list(getattr(dump, "locatable", None) or []), - window=dict(getattr(dump, "window", None) or {}), - ) - except Exception: - pass - - def _seed_windows_element_atlas(self) -> None: - """Reuse recent captures as the event-time element repository. - - Values are discarded by ``learn_elements``; only stable names and - geometry are retained. This lets a new recording identify custom - editors before their slow accessibility provider completes a new dump. - """ - recorder = self._recorder - if recorder is None or not hasattr(recorder, "learn_elements"): - return - try: - siblings = sorted( - ( - path for path in self.flow_dir.parent.iterdir() - if path.is_dir() and path != self.flow_dir - ), - key=lambda path: path.stat().st_mtime, - reverse=True, - )[:8] - except Exception: - return - learned_classes: set[str] = set() - for sibling in siblings: - observations = sibling / "observations" - if not observations.is_dir(): - continue - try: - metas = sorted( - observations.glob("*/meta.json"), - key=lambda path: path.stat().st_mtime, - reverse=True, - ) - except Exception: - continue - for meta_path in metas: - try: - import json - - meta = json.loads(meta_path.read_text(encoding="utf-8")) - if str(meta.get("surface") or "") != "windows": - continue - selection = dict(meta.get("selection") or {}) - class_name = str( - selection.get("chosen_class") - or selection.get("event_class") - or "" - ) - if not class_name or class_name in learned_classes: - continue - locatable_path = meta_path.parent / "locatable.json" - rows = json.loads(locatable_path.read_text(encoding="utf-8")) - recorder.learn_elements( - rows, - window={ - "class_name": class_name, - "hwnd": int(selection.get("chosen_hwnd") or 0), - "pid": int(selection.get("chosen_pid") or 0), - "title": str(selection.get("event_title") or ""), - }, - ) - learned_classes.add(class_name) - except Exception: - continue - if "WeWorkWindow" in learned_classes and "SearchResultWindow2" in learned_classes: - break - - def pause(self) -> None: - if self._recorder: - self._recorder.pause() - self._submit("pause") - - def resume(self) -> None: - if self._recorder: - self._recorder.resume() - self._submit("resume") - - def stop(self) -> None: - recorder_stop_error: Optional[BaseException] = None - if self._recorder: - try: - self._recorder.stop() - except Exception as exc: - recorder_stop_error = exc - if hasattr(self._recorder, "health"): - try: - self._recorder_health = self._recorder.health() - except Exception as exc: - self._recorder_health = {"last_error": str(exc)} - self._recorder = None - if self._event_thread and self._event_thread.is_alive(): - self._submit("stop") - self._event_thread.join(timeout=35.0) - if self._event_thread.is_alive(): - self.manager.last_warning = "event consumer did not stop within 35s" - raise RuntimeError(self.manager.last_warning) - else: - self._event_thread = None - self._event_stop.set() - if self._evidence is not None: - self._evidence.stop(timeout=3.0) - self.manager.stop() - # Point flow.json v2 fields (already present from create_flow). - try: - flow = art.load_flow(self.flow_dir) - v2 = dict(flow.get("v2") or {}) - v2["action_count"] = len(self.manager.action_flow.actions) - v2["state_count"] = len(self.manager.action_flow.states) - v2["observation_count"] = len( - self.manager.snapshots.observation_ids() - ) - v2["element_count"] = len(self.manager.catalog.catalog.elements) - v2["raw_event_journal"] = "journal/raw_events.jsonl" - v2["pages"] = "pages.json" - v2["diagnostics_dir"] = "diagnostics/" - v2["validation"] = "diagnostics/validation.json" - v2["replay_report"] = "diagnostics/replay_report.json" - flow["v2"] = v2 - art.save_flow(self.flow_dir, flow) - except Exception: - pass - self._write_diagnostics() - self._refresh_stats() - if recorder_stop_error is not None: - self.manager.last_warning = str(recorder_stop_error) - raise RuntimeError(str(recorder_stop_error)) from recorder_stop_error - - def checkpoint(self) -> Any: - return self.manager.record_checkpoint() - - def heartbeat(self) -> None: - self._submit("heartbeat") - - def stats(self) -> dict[str, Any]: - with self._stats_lock: - return dict(self._stats_cache) - - def _refresh_stats(self) -> None: - st = self.manager.stats() - st["surface_recorder"] = ( - self._recorder.status if self._recorder else "off" - ) - st["queued_events"] = self._event_queue.qsize() - st["dropped_events"] = self._dropped_events - if self._recorder is not None and hasattr(self._recorder, "health"): - try: - st["input_backend"] = self._recorder.health() - except Exception as exc: - st["input_backend"] = {"last_error": str(exc)} - elif self._recorder_health: - st["input_backend"] = dict(self._recorder_health) - if self._evidence is not None: - st["evidence_capture_count"] = self._evidence.capture_count - st["evidence_dropped"] = self._evidence.dropped - st["evidence_queued"] = self._evidence._queue.qsize() - st["transient_evidence_queued"] = self._evidence._urgent_queue.qsize() - st["evidence_failures"] = self._evidence.failure_count - with self._stats_lock: - self._stats_cache = st - - def _write_diagnostics(self) -> None: - pages: dict[str, dict[str, Any]] = {} - lags: list[int] = [] - for action in self.manager._actions_by_id.values(): - raw = dict(action.raw_event or {}) - meta = dict(raw.get("meta") or {}) - page_id = str(meta.get("page_id") or "") - if page_id: - page = pages.setdefault(page_id, {"page_id": page_id, "urls": [], "document_ids": []}) - url = str(raw.get("url") or (action.input or {}).get("url") or "") - document_id = str(meta.get("document_id") or "") - if url and url not in page["urls"]: - page["urls"].append(url) - if document_id and document_id not in page["document_ids"]: - page["document_ids"].append(document_id) - lag = (action.evidence or {}).get("evidence_lag_ms") - if isinstance(lag, int): - lags.append(lag) - diagnostics = self.flow_dir / "diagnostics" - diagnostics.mkdir(parents=True, exist_ok=True) - try: - art.write_json(self.flow_dir / "pages.json", { - "schema": "openclaw.ui_inspector.pages.v1", - "pages": list(pages.values()), - }) - recorder_health = dict(self._recorder_health) - if self._recorder is not None and hasattr(self._recorder, "health"): - try: - recorder_health = self._recorder.health() - except Exception as exc: - recorder_health = {"last_error": str(exc)} - art.write_json(diagnostics / "metrics.json", { - "schema": "openclaw.ui_inspector.metrics.v1", - "raw_event_count": self.journal._sequence, - "action_count": len(self.manager.action_flow.actions), - "state_count": len(self.manager.action_flow.states), - "observation_count": len(self.manager.snapshots.observation_ids()), - "element_count": len(self.manager.catalog.catalog.elements), - "event_queue_dropped": self._dropped_events, - "evidence_queue_dropped": self._evidence.dropped if self._evidence else 0, - "evidence_capture_count": self._evidence.capture_count if self._evidence else 0, - "evidence_failure_count": self._evidence.failure_count if self._evidence else 0, - "evidence_lag_ms_max": max(lags) if lags else None, - "evidence_lag_ms_average": round(sum(lags) / len(lags), 2) if lags else None, - "input_backend": recorder_health, - }) - from diagnostics import validate_capture, write_replay_preflight - - art.write_json(diagnostics / "validation.json", validate_capture(self.flow_dir)) - write_replay_preflight(self.flow_dir) - except Exception as exc: - self.manager.last_warning = f"diagnostics write failed: {exc}" diff --git a/tools/ui_inspector/kit_browser.py b/tools/ui_inspector/kit_browser.py deleted file mode 100644 index b39df20..0000000 --- a/tools/ui_inspector/kit_browser.py +++ /dev/null @@ -1,729 +0,0 @@ -"""Inspector-owned Playwright browser session (dev-time page dump). - -All Playwright sync API calls run on one dedicated worker thread -(Playwright sync is not cross-thread safe). - -Dump / screenshot / recorder target the **currently visible** page -(document.visibilityState === 'visible'), not a sticky pages[0]. -This keeps list ↔ detail tab switching aligned with what the user sees. - -Uses jiangchang_skill_core.rpa stealth launch parts + dump_page_sync. -Profile lives under ui_inspector/_browser_profile (not account-manager). -""" - -from __future__ import annotations - -import atexit -import queue -import threading -import uuid -from pathlib import Path -from typing import Any, Callable, Optional - -_HERE = Path(__file__).resolve().parent -_DEFAULT_PROFILE = _HERE / "_browser_profile" -_DEFAULT_START_URL = "about:blank" - -_cmd_q: queue.Queue = queue.Queue() -_worker: Optional[threading.Thread] = None -_worker_lock = threading.Lock() - -# Owned exclusively by worker thread: -_pw = None -_context = None -_page = None -_started = False -_bridge_installed = False -_bridge_sink: Optional[Callable[[dict[str, Any]], None]] = None -_bridge_lock = threading.Lock() -_BRIDGE_NAME = "__openclawUiInspectorEmit" -_page_ids: dict[int, str] = {} -_page_refs: dict[int, Any] = {} -_pages_by_id: dict[str, Any] = {} -_frame_ids: dict[int, str] = {} -_frame_refs: dict[int, Any] = {} - - -def _page_id_for(page: Any) -> str: - if not _page_is_open(page): - return "" - key = id(page) - page_id = _page_ids.get(key) if _page_refs.get(key) is page else None - if not page_id: - page_id = f"page_{uuid.uuid4().hex[:12]}" - _page_ids[key] = page_id - _page_refs[key] = page - _pages_by_id[page_id] = page - return page_id - - -def _frame_id_for(frame: Any, *, page_id: str = "") -> str: - if frame is None: - return "" - key = id(frame) - frame_id = _frame_ids.get(key) if _frame_refs.get(key) is frame else None - if not frame_id: - prefix = page_id or "page" - frame_id = f"{prefix}:frame_{uuid.uuid4().hex[:10]}" - _frame_ids[key] = frame_id - _frame_refs[key] = frame - return frame_id - - -def _page_for_id(page_id: str) -> Any: - page = _pages_by_id.get(str(page_id or "")) - if _page_is_open(page): - return page - if page_id: - _pages_by_id.pop(str(page_id), None) - return None - - -def kit_page_dump_available() -> bool: - try: - from jiangchang_skill_core.rpa.page_dump import dump_page_sync # noqa: F401 - - return True - except ImportError: - return False - - -def profile_dir() -> Path: - return _DEFAULT_PROFILE - - -def _ensure_worker() -> None: - global _worker - with _worker_lock: - if _worker is not None and _worker.is_alive(): - return - - def loop() -> None: - while True: - item = _cmd_q.get() - if item is None: - break - fn, args, kwargs, out_q = item - try: - out_q.put(("ok", fn(*args, **kwargs))) - except BaseException as exc: - out_q.put(("err", exc)) - - _worker = threading.Thread(target=loop, name="ui-inspector-pw", daemon=True) - _worker.start() - atexit.register(shutdown) - - -def _call_on_worker( - fn: Callable[..., Any], - *args: Any, - timeout_sec: float = 30.0, - **kwargs: Any, -) -> Any: - _ensure_worker() - out_q: queue.Queue = queue.Queue() - _cmd_q.put((fn, args, kwargs, out_q)) - try: - status, payload = out_q.get(timeout=max(0.1, float(timeout_sec))) - except queue.Empty as exc: - raise RuntimeError( - f"ERROR:BROWSER_WORKER_TIMEOUT Playwright 命令超过 {timeout_sec:g} 秒未返回。" - ) from exc - if status == "err": - raise payload - return payload - - -def _teardown_unlocked() -> None: - global _pw, _context, _page, _started, _bridge_installed, _bridge_sink - try: - if _context is not None: - _context.close() - except Exception: - pass - try: - if _pw is not None: - _pw.stop() - except Exception: - pass - _pw = None - _context = None - _page = None - _started = False - _bridge_installed = False - with _bridge_lock: - _bridge_sink = None - _page_ids.clear() - _page_refs.clear() - _pages_by_id.clear() - _frame_ids.clear() - _frame_refs.clear() - - -def _goto_safe(page: Any, start_url: str) -> dict[str, Any]: - url = (start_url or "").strip() - if not url or url == "about:blank": - return {"ok": True, "requested_url": url, "actual_url": page.url or ""} - try: - page.goto(url, wait_until="domcontentloaded", timeout=60_000) - except Exception as exc: - actual = "" - try: - actual = str(page.url or "") - except Exception: - pass - # Some sites keep loading after domcontentloaded timeout. A real HTTP - # page is usable, but the warning must remain visible to callers. - if actual.startswith(("http://", "https://")): - return { - "ok": True, - "requested_url": url, - "actual_url": actual, - "warning": str(exc), - } - raise RuntimeError( - f"ERROR:NAVIGATION_FAILED requested={url!r} actual={actual!r}: {exc}" - ) from exc - actual = str(page.url or "") - if not actual or actual == "about:blank" or actual.startswith("about:"): - raise RuntimeError( - f"ERROR:NAVIGATION_FAILED requested={url!r} actual={actual!r}" - ) - return {"ok": True, "requested_url": url, "actual_url": actual} - - -def _page_is_open(page: Any) -> bool: - try: - return page is not None and not page.is_closed() - except Exception: - return False - - -def _open_pages(context: Any) -> list[Any]: - if context is None: - return [] - out: list[Any] = [] - try: - pages = list(context.pages or []) - except Exception: - return [] - for p in pages: - if _page_is_open(p): - _page_id_for(p) - out.append(p) - return out - - -def _page_visibility(page: Any) -> str: - """Return document.visibilityState or '' on failure.""" - if not _page_is_open(page): - return "" - try: - state = page.evaluate("() => document.visibilityState") - return str(state or "").strip().lower() - except Exception: - return "" - - -def resolve_active_page( - context: Any, - *, - preferred: Any = None, -) -> tuple[Any, dict[str, Any]]: - """Pick the page the user is looking at (visible tab). - - Returns (page, meta). Prefer visibilityState===visible; fall back to - preferred if still open, else first open page. - """ - pages = _open_pages(context) - meta: dict[str, Any] = { - "resolve": "none", - "page_count": len(pages), - "visible_count": 0, - } - if not pages: - raise RuntimeError("ERROR:NO_OPEN_PAGE Playwright 会话中没有可用标签页。") - - visible: list[Any] = [] - for p in pages: - if _page_visibility(p) == "visible": - visible.append(p) - meta["visible_count"] = len(visible) - - if len(visible) == 1: - meta["resolve"] = "visibility" - return visible[0], meta - if len(visible) > 1: - # Multiple windows each with a visible tab — prefer preferred if among them - if preferred is not None and preferred in visible: - meta["resolve"] = "visibility_preferred" - return preferred, meta - meta["resolve"] = "visibility_first" - return visible[0], meta - - if preferred is not None and _page_is_open(preferred) and preferred in pages: - meta["resolve"] = "preferred_fallback" - return preferred, meta - - meta["resolve"] = "first_open" - return pages[0], meta - - -def _sync_active_page(*, update_global: bool = True) -> tuple[Any, dict[str, Any]]: - """Resolve visible page; optionally assign to module ``_page``.""" - global _page - if _context is None: - raise RuntimeError("ERROR:NO_BROWSER_SESSION Playwright 会话未启动。") - page, meta = resolve_active_page(_context, preferred=_page) - if update_global: - _page = page - try: - meta["url"] = page.url or "" - meta["title"] = page.title() or "" - except Exception: - meta["url"] = "" - meta["title"] = "" - meta["page_id"] = _page_id_for(page) - return page, meta - - -def _resolve_capture_page(page_id: str = "") -> tuple[Any, dict[str, Any]]: - global _page - requested = str(page_id or "") - if requested: - page = _page_for_id(requested) - if page is not None: - _page = page - meta = { - "resolve": "event_page_id", - "page_id": requested, - "page_count": len(_open_pages(_context)), - "visible_count": None, - } - try: - meta["url"] = page.url or "" - meta["title"] = page.title() or "" - except Exception: - meta["url"] = "" - meta["title"] = "" - return page, meta - raise RuntimeError(f"ERROR:EVENT_PAGE_CLOSED page_id={requested}") - return _sync_active_page(update_global=True) - - -def _ensure_session_worker( - *, - start_url: str = _DEFAULT_START_URL, - force_goto: bool = False, -) -> dict[str, Any]: - global _pw, _context, _page, _started - want = (start_url or "").strip() or _DEFAULT_START_URL - - if _started and _context is not None: - try: - navigation = None - pages = _open_pages(_context) - if not pages: - _teardown_unlocked() - else: - # Start-flow navigation: prefer first open tab (stable home), - # not whatever detail tab happens to be visible. - if force_goto and want and want != "about:blank": - home = pages[0] - cur = "" - try: - cur = home.url or "" - except Exception: - cur = "" - if cur.rstrip("/") != want.rstrip("/"): - navigation = _goto_safe(home, want) - _page = home - else: - _page, _meta = _sync_active_page(update_global=True) - if want and want != "about:blank" and ( - (_page.url or "") in {"", "about:blank"} - or str(_page.url or "").startswith("about:") - ): - navigation = _goto_safe(_page, want) - _page, _meta = _sync_active_page(update_global=True) - try: - live_url = _page.url or "" - except Exception: - live_url = "" - return { - "ok": True, - "reused": True, - "url": live_url, - "profile_dir": str(profile_dir()), - "page_count": len(_open_pages(_context)), - "navigation": navigation, - } - except Exception: - _teardown_unlocked() - - if not kit_page_dump_available(): - raise RuntimeError( - "ERROR:KIT_PAGE_DUMP_MISSING 需要 jiangchang-platform-kit>=1.2.11 " - "(jiangchang_skill_core.rpa.page_dump)。请升级 kit 后重试。" - ) - - try: - from playwright.sync_api import sync_playwright - except ImportError as exc: - raise RuntimeError( - "ERROR:MISSING_PLAYWRIGHT 未安装 playwright。" - "请在与技能相同的 Python 环境中安装后重试。" - ) from exc - - from jiangchang_skill_core.runtime_env import find_chrome_executable - from jiangchang_skill_core.rpa.stealth import ( - STEALTH_INIT_SCRIPT, - persistent_context_launch_parts, - stealth_enabled, - ) - - chrome = find_chrome_executable() - if not chrome: - raise RuntimeError( - "ERROR:MISSING_BROWSER 未检测到 Chrome 或 Edge，请安装后重试。" - ) - - prof = profile_dir() - prof.mkdir(parents=True, exist_ok=True) - args, ignore = persistent_context_launch_parts(extra_args=["--start-maximized"]) - launch_kwargs: dict[str, Any] = dict( - user_data_dir=str(prof), - headless=False, - executable_path=chrome, - locale="zh-CN", - no_viewport=True, - args=args, - ) - if ignore is not None: - launch_kwargs["ignore_default_args"] = ignore - - _pw = sync_playwright().start() - try: - _context = _pw.chromium.launch_persistent_context(**launch_kwargs) - if stealth_enabled(): - _context.add_init_script(STEALTH_INIT_SCRIPT) - if _context.pages: - _page = _context.pages[0] - else: - _page = _context.new_page() - navigation = None - if want and want != "about:blank": - navigation = _goto_safe(_page, want) - elif want and ( - _page.url in {"", "about:blank"} or str(_page.url).startswith("about:") - ): - navigation = _goto_safe(_page, want) - _started = True - return { - "ok": True, - "reused": False, - "url": _page.url, - "profile_dir": str(prof), - "page_count": len(_open_pages(_context)), - "navigation": navigation, - } - except Exception: - _teardown_unlocked() - raise - - -def _dump_worker(*, max_nodes: int, locatable_limit: int, page_id: str = "") -> dict[str, Any]: - from jiangchang_skill_core.rpa.page_dump import dump_page_sync - - _ensure_session_worker() - page, meta = _resolve_capture_page(page_id) - result = dump_page_sync(page, max_nodes=max_nodes, locatable_limit=locatable_limit) - stats = dict(result.get("stats") or {}) - stats["active_page"] = meta - result["stats"] = stats - return result - - -def _screenshot_worker(*, page_id: str = "") -> bytes: - _ensure_session_worker() - page, _meta = _resolve_capture_page(page_id) - return page.screenshot(type="png", full_page=False) - - -def _session_info_worker() -> dict[str, Any]: - if not _started or _context is None: - return {"active": False} - try: - page, meta = _sync_active_page(update_global=True) - return { - "active": True, - "url": page.url, - "title": page.title(), - "profile_dir": str(profile_dir()), - "page_count": meta.get("page_count"), - "resolve": meta.get("resolve"), - "page_id": meta.get("page_id"), - } - except Exception as exc: - return {"active": False, "error": str(exc)} - - -def _dispatch_bridge_event(source: Any, payload: Any) -> None: - if not isinstance(payload, dict): - return - item = dict(payload) - try: - page = source.get("page") if isinstance(source, dict) else None - frame = source.get("frame") if isinstance(source, dict) else None - page_id = _page_id_for(page) - item["page_id"] = page_id - item["frame_id"] = _frame_id_for(frame, page_id=page_id) - except Exception: - pass - with _bridge_lock: - sink = _bridge_sink - if sink is not None: - try: - sink(item) - except Exception: - pass - - -def _install_event_bridge_worker( - *, init_script: str, sink: Callable[[dict[str, Any]], None] -) -> dict[str, Any]: - global _bridge_installed, _bridge_sink - _ensure_session_worker() - assert _context is not None - with _bridge_lock: - _bridge_sink = sink - if not _bridge_installed: - _context.expose_binding(_BRIDGE_NAME, _dispatch_bridge_event) - _bridge_installed = True - _context.add_init_script(init_script) - installed = 0 - for page in _open_pages(_context): - try: - page.evaluate(init_script) - installed += 1 - except Exception: - pass - return {"ok": True, "binding": _BRIDGE_NAME, "installed_pages": installed} - - -def install_event_bridge( - init_script: str, sink: Callable[[dict[str, Any]], None] -) -> dict[str, Any]: - """Install a context-level recorder bridge before user actions begin.""" - return _call_on_worker( - _install_event_bridge_worker, - init_script=init_script, - sink=sink, - timeout_sec=15.0, - ) - - -def clear_event_bridge() -> None: - global _bridge_sink - with _bridge_lock: - _bridge_sink = None - - -def ensure_session( - *, - start_url: str = _DEFAULT_START_URL, - force_goto: bool = False, -) -> dict[str, Any]: - """Start or reuse headed persistent context; return status dict. - - force_goto=True: navigate to start_url even when session is reused - (used when「开始流程」带起始 URL). - """ - return _call_on_worker( - _ensure_session_worker, - start_url=start_url, - force_goto=force_goto, - timeout_sec=75.0, - ) - - -def session_info() -> dict[str, Any]: - return _call_on_worker(_session_info_worker, timeout_sec=10.0) - - -def dump_current_page( - *, max_nodes: int = 8000, locatable_limit: int = 300, page_id: str = "" -) -> dict[str, Any]: - """Dump an event-origin page, or resolve the active page as fallback.""" - return _call_on_worker( - _dump_worker, - max_nodes=max_nodes, - locatable_limit=locatable_limit, - page_id=page_id, - timeout_sec=30.0, - ) - - -def screenshot_png(*, page_id: str = "") -> bytes: - return _call_on_worker(_screenshot_worker, page_id=page_id, timeout_sec=20.0) - - -def page_id_for(page: Any) -> str: - """Return the stable id for a Page while already on the browser worker.""" - return _page_id_for(page) - - -def _validate_element_worker( - *, page_id: str, element: dict[str, Any], frame: Optional[dict[str, Any]] = None -) -> dict[str, int]: - page, _meta = _resolve_capture_page(page_id) - scope: Any = page - frame_meta = dict(frame or {}) - frame_url = str(frame_meta.get("url") or "") - frame_name = str(frame_meta.get("name") or "") - if frame_url or frame_name: - try: - for candidate in page.frames: - if frame_name and candidate.name == frame_name: - scope = candidate - break - if frame_url and candidate.url == frame_url: - scope = candidate - break - except Exception: - pass - - el = dict(element or {}) - role = str(el.get("role") or "").strip() - name = str(el.get("name") or "").strip() - test_id = str(el.get("test_id") or "").strip() - label = str(el.get("label") or "").strip() - placeholder = str(el.get("placeholder") or "").strip() - name_attr = str(el.get("name_attr") or "").strip() - href = str(el.get("href") or "").strip() - text_value = str(el.get("text") or "").strip() - css_id = str(el.get("id") or "").strip() - css_path = str(el.get("css") or el.get("css_path") or "").strip() - stable_css = str(el.get("stable_css") or "").strip() - semantic_role = str(el.get("semantic_role") or "").strip() - semantic_name = str(el.get("semantic_name") or "").strip() - counts: dict[str, int] = {} - - def count(strategy: str, build: Callable[[], Any]) -> None: - try: - counts[strategy] = int(build().count()) - except Exception: - pass - - def count_actionable(strategy: str, build: Callable[[], Any]) -> None: - try: - locator = build() - total = min(int(locator.count()), 50) - actionable = 0 - for index in range(total): - item = locator.nth(index) - if item.is_visible() and item.is_enabled(): - actionable += 1 - counts[strategy] = actionable - except Exception: - pass - - if role and name: - # Playwright accessible names normalize whitespace (SVG/icon buttons are - # a common example), while DOM innerText may not. Non-exact still - # yields a trustworthy uniqueness count and mirrors getByRole defaults. - count("role", lambda: scope.get_by_role(role, name=name, exact=False)) - elif role and bool(el.get("contenteditable")): - count("role", lambda: scope.get_by_role(role)) - if semantic_role and semantic_name: - count_actionable( - "semantic_role", - lambda: scope.get_by_role( - semantic_role, name=semantic_name, exact=False - ), - ) - if test_id: - count("test_id", lambda: scope.get_by_test_id(test_id)) - if label: - count("label", lambda: scope.get_by_label(label, exact=True)) - if placeholder: - count("placeholder", lambda: scope.get_by_placeholder(placeholder, exact=True)) - if name_attr: - escaped_name = name_attr.replace("\\", "\\\\").replace('"', '\\"') - count("name_attr", lambda: scope.locator(f'[name="{escaped_name}"]')) - if href: - escaped_href = href.replace("\\", "\\\\").replace('"', '\\"') - count("href", lambda: scope.locator(f'a[href="{escaped_href}"]')) - if text_value and len(text_value) <= 80: - count("text", lambda: scope.get_by_text(text_value, exact=True)) - if css_id and not css_id[0].isdigit(): - escaped = css_id.replace("\\", "\\\\").replace('"', '\\"') - count("css_id", lambda: scope.locator(f'[id="{escaped}"]')) - if css_path: - count("css", lambda: scope.locator(css_path)) - if stable_css: - count("stable_css", lambda: scope.locator(stable_css)) - return counts - - -def validate_element_on_page( - *, page_id: str, element: dict[str, Any], frame: Optional[dict[str, Any]] = None -) -> dict[str, int]: - """Count locator candidates against the exact live event-origin page.""" - if not page_id: - return {} - return _call_on_worker( - _validate_element_worker, - page_id=page_id, - element=element, - frame=frame, - timeout_sec=10.0, - ) - - -def call_on_page(fn: Callable[[Any], Any]) -> Any: - """Run ``fn(page)`` on the **visible** Playwright page (worker thread).""" - - def _run() -> Any: - _ensure_session_worker() - page, _meta = _sync_active_page(update_global=True) - return fn(page) - - return _call_on_worker(_run, timeout_sec=30.0) - - -def call_on_each_open_page(fn: Callable[[Any], Any]) -> list[Any]: - """Run ``fn(page)`` for every open tab; returns list of per-page results.""" - - def _run() -> list[Any]: - _ensure_session_worker() - assert _context is not None - pages = _open_pages(_context) - # Keep _page synced to visible for dump consistency - try: - _sync_active_page(update_global=True) - except Exception: - pass - results: list[Any] = [] - for p in pages: - try: - results.append(fn(p)) - except Exception as exc: - results.append({"error": str(exc)}) - return results - - return _call_on_worker(_run, timeout_sec=20.0) - - -def shutdown() -> None: - def _stop() -> None: - _teardown_unlocked() - - try: - _call_on_worker(_stop, timeout_sec=5.0) - except Exception: - pass - try: - _cmd_q.put(None) - except Exception: - pass diff --git a/tools/ui_inspector/kit_wake.py b/tools/ui_inspector/kit_wake.py deleted file mode 100644 index abbf0c6..0000000 --- a/tools/ui_inspector/kit_wake.py +++ /dev/null @@ -1,284 +0,0 @@ -"""调用 jiangchang_desktop_wake（只调不复制注入逻辑）。 - -- 主窗 WeWorkWindow → ``ensure_ui_ready("wecom")`` -- 搜索浮层 SearchResultWindow2 → ``ensure_ui_ready("wecom_search")`` -未安装 kit 时降级为跳过，不阻断采集。 - -Warm cache key = profile + hwnd（经 WeCom Adapter），避免主窗 warm -状态错误代表 SearchResultWindow2 已暖。 -""" - -from __future__ import annotations - -import time -from typing import Any, Optional - -# A successful provider probe remains valid for the lifetime of a normal -# recording flow. The old 45 second TTL made long desktop flows re-run an -# expensive WeCom wake in the middle of recording and produced minute-late -# evidence. -_CACHE_TTL_SEC = 600.0 - -# cache_key -> cache (profile:hwnd:N) -_caches: dict[str, dict[str, Any]] = {} - -_PROFILE_WECOM = "wecom" -_PROFILE_WECOM_SEARCH = "wecom_search" - -_THRESHOLDS = { - _PROFILE_WECOM: 80, - _PROFILE_WECOM_SEARCH: 20, -} - - -def _wecom_adapter(): - try: - from adapters.wecom import wecom_adapter - - return wecom_adapter - except Exception: - try: - from .adapters.wecom import wecom_adapter # type: ignore - - return wecom_adapter - except Exception: - return None - - -def resolve_wake_profile(*, class_name: str = "", title: str = "") -> Optional[str]: - """返回应调用的 kit profile id；非企微相关则 None。""" - ad = _wecom_adapter() - if ad is not None: - return ad.wake_profile(class_name=class_name, title=title) - # Compatibility fallback if adapter import fails - cls = (class_name or "").strip() - title = title or "" - if cls == "SearchResultWindow2": - return _PROFILE_WECOM_SEARCH - if cls == "WeWorkWindow": - return _PROFILE_WECOM - if "WeWork" in cls and ("企业微信" in title or "WeCom" in title or "wecom" in title.lower()): - return _PROFILE_WECOM - return None - - -def looks_like_wecom(*, class_name: str = "", title: str = "") -> bool: - return resolve_wake_profile(class_name=class_name, title=title) is not None - - -def kit_available() -> bool: - try: - from jiangchang_desktop_wake import ensure_ui_ready # noqa: F401 - - return True - except ImportError: - return False - - -def _cache_key(profile: str, hwnd: Optional[int]) -> str: - ad = _wecom_adapter() - if ad is not None: - return ad.cache_key(profile=profile, hwnd=int(hwnd or 0)) - return f"{profile}:hwnd:{int(hwnd or 0)}" - - -def auto_ensure_if_wecom( - *, - class_name: str = "", - title: str = "", - hwnd: Optional[int] = None, - force: bool = False, - findall_hint: Optional[int] = None, -) -> dict[str, Any]: - """若目标像企微主窗/搜索浮层则静默 ensure；否则跳过。""" - profile = resolve_wake_profile(class_name=class_name, title=title) - if profile is None: - return { - "applicable": False, - "skipped": True, - "reason": "not_wecom", - "available": kit_available(), - } - - ad = _wecom_adapter() - warm_min = int(ad.warm_threshold(profile) if ad else _THRESHOLDS.get(profile, 80)) - key = _cache_key(profile, hwnd) - cache = _caches.setdefault( - key, {"t": 0.0, "warm": False, "hwnd": int(hwnd or 0), "findall": 0, "profile": profile} - ) - now = time.monotonic() - hint = int(findall_hint) if findall_hint is not None else -1 - # Require same hwnd in cache — main window warm must not cover search popup. - cached_hwnd = int(cache.get("hwnd") or 0) - same_window = (not hwnd) or (cached_hwnd == int(hwnd)) - if ( - not force - and same_window - and cache.get("warm") - and (now - float(cache.get("t") or 0.0)) < _CACHE_TTL_SEC - ): - return { - "applicable": True, - "available": True, - "skipped": True, - "reason": "cache_warm", - "ok": True, - "warm": True, - "wake_applied": False, - "findall": int(cache.get("findall") or 0), - "hwnd": int(cache.get("hwnd") or 0), - "profile": profile, - "cache_key": key, - "observed_findall_hint": hint, - } - - # A direct probe is authoritative. Do not run an invasive wake merely - # because this process did not create the cache entry itself. - if not force and hint >= warm_min: - cache.update( - {"t": now, "warm": True, "hwnd": int(hwnd or 0), "findall": hint} - ) - return { - "applicable": True, - "available": kit_available(), - "skipped": True, - "reason": "probe_warm", - "ok": True, - "warm": True, - "wake_applied": False, - "findall": hint, - "hwnd": int(hwnd or 0), - "profile": profile, - "cache_key": key, - } - - try: - from jiangchang_desktop_wake import ensure_ui_ready - from jiangchang_desktop_wake.errors import DesktopWakeError - except ImportError: - return { - "applicable": True, - "available": False, - "skipped": True, - "reason": "kit_not_installed", - "ok": False, - "profile": profile, - "hint": "请安装 jiangchang-platform-kit>=1.2.6（含 wecom_search）后再采企微", - } - - t0 = time.perf_counter() - try: - result = ensure_ui_ready( - profile, - hwnd=int(hwnd) if hwnd else None, - raise_on_cold=True, - ) - except DesktopWakeError as exc: - raw = (getattr(exc, "code", None) or "").strip() - code = raw[6:] if raw.startswith("ERROR:") else (raw or "WAKE_FAILED") - return { - "applicable": True, - "available": True, - "skipped": False, - "ok": False, - "warm": False, - "wake_applied": False, - "error_code": code, - "message": str(getattr(exc, "message", None) or exc), - "elapsed_ms": int(round((time.perf_counter() - t0) * 1000)), - "profile": profile, - } - except Exception as exc: - return { - "applicable": True, - "available": True, - "skipped": False, - "ok": False, - "warm": False, - "error_code": "ENSURE_EXCEPTION", - "message": f"{type(exc).__name__}: {exc}", - "elapsed_ms": int(round((time.perf_counter() - t0) * 1000)), - "profile": profile, - } - - after = result.probe_after - findall = int(after.findall_count) if after is not None else 0 - report = { - "applicable": True, - "available": True, - "skipped": False, - "ok": bool(result.warm), - "warm": bool(result.warm), - "wake_applied": bool(result.wake_applied), - "findall": findall, - "hwnd": int(result.hwnd) if result.hwnd else 0, - "elapsed_ms": int(round((time.perf_counter() - t0) * 1000)), - "profile": profile, - "markers_found": list(after.markers_found) if after is not None else [], - } - report["cache_key"] = key - if result.warm: - # Persist under result hwnd so main≠search caches stay separate. - final_key = _cache_key(profile, report["hwnd"] or hwnd) - final_cache = _caches.setdefault( - final_key, - {"t": 0.0, "warm": False, "hwnd": int(report["hwnd"] or 0), "findall": 0, "profile": profile}, - ) - final_cache["t"] = now - final_cache["warm"] = True - final_cache["hwnd"] = int(report["hwnd"] or hwnd or 0) - final_cache["findall"] = findall - report["cache_key"] = final_key - else: - cache["warm"] = False - return report - - -def note_provider_warm( - *, class_name: str = "", title: str = "", hwnd: int = 0, findall: int = 0 -) -> bool: - """Pin a proven-warm provider so recorder evidence never re-wakes it.""" - profile = resolve_wake_profile(class_name=class_name, title=title) - adapter = _wecom_adapter() - threshold = int( - adapter.warm_threshold(profile) - if adapter and profile - else _THRESHOLDS.get(profile or "", 80) - ) - count = int(findall or 0) - if not profile or count < threshold: - return False - key = _cache_key(profile, hwnd) - _caches[key] = { - "t": time.monotonic(), - "warm": True, - "hwnd": int(hwnd or 0), - "findall": count, - "profile": profile, - } - return True - - -def format_kit_wake_line(report: Optional[dict[str, Any]]) -> str: - if not report: - return "kit_wake: (none)" - if not report.get("applicable"): - return "kit_wake: n/a (非企微)" - if report.get("reason") == "kit_not_installed": - return "kit_wake: 未安装 platform-kit（企微可能冷树）" - prof = report.get("profile") or "?" - if report.get("skipped") and report.get("reason") == "cache_warm": - return ( - f"kit_wake[{prof}]: cache warm=True findall={report.get('findall')} " - f"hwnd={report.get('hwnd')}" - ) - if report.get("ok"): - return ( - f"kit_wake[{prof}]: warm={report.get('warm')} " - f"wake_applied={report.get('wake_applied')} " - f"findall={report.get('findall')} elapsed_ms={report.get('elapsed_ms')}" - ) - return ( - f"kit_wake[{prof}]: FAILED code={report.get('error_code')} " - f"msg={report.get('message')}" - ) diff --git a/tools/ui_inspector/locators/__init__.py b/tools/ui_inspector/locators/__init__.py deleted file mode 100644 index 178105c..0000000 --- a/tools/ui_inspector/locators/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Locator compilers + validators + element catalog.""" - -from .browser import compile_browser_locators -from .catalog import ElementCatalogService -from .validator import validate_locator_against_rows -from .windows import compile_windows_locators - -__all__ = [ - "compile_browser_locators", - "compile_windows_locators", - "validate_locator_against_rows", - "ElementCatalogService", -] diff --git a/tools/ui_inspector/locators/base.py b/tools/ui_inspector/locators/base.py deleted file mode 100644 index 78f7517..0000000 --- a/tools/ui_inspector/locators/base.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Shared locator compiler helpers.""" - -from __future__ import annotations - -from typing import Any, Optional - -from models.locator import LocatorCandidate - - -def rank_and_select_primary( - candidates: list[LocatorCandidate], -) -> tuple[Optional[LocatorCandidate], list[LocatorCandidate]]: - """ - Pick production primary only if unique and not coordinate-based. - Fallback list excludes primary. - """ - ranked = sorted( - candidates, - key=lambda c: ( - 0 if c.production_eligible else 1, - 0 if c.unique else 1, - 0 if c.uses_coordinates else 0, - 1 if c.uses_coordinates else 0, - 1 if c.uses_index else 0, - -float(c.stability_score or 0.0), - ), - ) - primary: Optional[LocatorCandidate] = None - for c in ranked: - if c.production_eligible and not c.uses_coordinates: - primary = c - break - fallbacks = [c for c in ranked if c is not primary] - return primary, fallbacks - - -def count_matches( - rows: list[dict[str, Any]], - *, - properties: dict[str, Any], - scope: Optional[dict[str, Any]] = None, -) -> int: - """Count rows matching structured properties (snapshot / live tree).""" - props = {k: v for k, v in (properties or {}).items() if v not in (None, "")} - if not props: - return 0 - scope = scope or {} - n = 0 - for r in rows or []: - if not isinstance(r, dict): - continue - if scope.get("window_class"): - # optional soft filter if row carries window info - rc = r.get("window_class") or r.get("window_class_name") - if rc and rc != scope.get("window_class"): - continue - ok = True - for k, v in props.items(): - if k == "control_type": - rv = r.get("control_type") or r.get("role") or "" - elif k == "role": - rv = r.get("role") or r.get("control_type") or "" - elif k == "text": - rv = r.get("text") or r.get("value") or r.get("name") or "" - elif k == "test_id": - rv = r.get("test_id") or r.get("automation_id") or "" - else: - rv = r.get(k) - if str(rv or "") != str(v): - ok = False - break - if ok: - n += 1 - return n diff --git a/tools/ui_inspector/locators/browser.py b/tools/ui_inspector/locators/browser.py deleted file mode 100644 index 69c2e42..0000000 --- a/tools/ui_inspector/locators/browser.py +++ /dev/null @@ -1,190 +0,0 @@ -"""Browser locator compiler (role / test id / label / placeholder / text / css id).""" - -from __future__ import annotations - -from typing import Any, Optional - -from models.locator import LocatorCandidate - -from .base import count_matches, rank_and_select_primary - - -def _expr(strategy: str, props: dict[str, Any]) -> str: - parts = [f"{k}={v!r}" for k, v in props.items() if v not in (None, "")] - return f"{strategy}:" + ",".join(parts) - - -def compile_browser_locators( - element: dict[str, Any], - *, - rows: Optional[list[dict[str, Any]]] = None, - frame: Optional[dict[str, Any]] = None, -) -> list[LocatorCandidate]: - """ - Generate ordered Playwright-oriented locator candidates. - - Priority roughly: role+name → test id → label → placeholder → text → css id → scoped. - Coordinates are never emitted as production candidates. - """ - el = element or {} - rows = list(rows or []) - scope = {"frame_url": (frame or {}).get("url") or "", "frame_name": (frame or {}).get("name") or ""} - cands: list[LocatorCandidate] = [] - - role = str(el.get("role") or el.get("control_type") or "").strip() - name = str(el.get("name") or "").strip() - text = str(el.get("text") or el.get("value") or "").strip() - is_editable = bool(el.get("contenteditable")) - if is_editable: - # Draft.js/contenteditable often exposes current user text as its - # accessible name. That value is payload, never element identity. - current_value = str(el.get("value") or "").strip() - if name == current_value or name == text: - name = "" - text = "" - test_id = str(el.get("test_id") or el.get("data_testid") or el.get("automation_id") or "").strip() - label = str(el.get("label") or "").strip() - placeholder = str(el.get("placeholder") or "").strip() - name_attr = str(el.get("name_attr") or "").strip() - href = str(el.get("href") or "").strip() - css_id = str(el.get("id") or el.get("css_id") or "").strip() - css_path = str(el.get("css") or el.get("css_path") or "").strip() - stable_css = str(el.get("stable_css") or "").strip() - semantic_role = str(el.get("semantic_role") or "").strip() - semantic_name = str(el.get("semantic_name") or "").strip() - # Prefer existing primary_locator from page_dump if present - existing = el.get("primary_locator") or {} - if isinstance(existing, dict) and existing.get("expr"): - props = dict(existing.get("properties") or {}) - if not props and existing.get("strategy"): - props = {"expr": existing.get("expr")} - c = LocatorCandidate( - strategy=str(existing.get("strategy") or "primary"), - scope=scope, - properties=props or {"expression": existing.get("expr")}, - expression=str(existing.get("expr") or ""), - stability_score=0.85, - reasons=["from_page_dump_primary_locator"], - ) - if rows and props: - c.mark_counts(count_matches(rows, properties=props, scope=scope)) - c.validation = "snapshot_only" - elif rows and existing.get("expr"): - # Cannot count without structured props — leave unique unknown - c.validation = "snapshot_only" - cands.append(c) - - for pw in el.get("playwright_candidates") or []: - if not isinstance(pw, dict): - continue - props = dict(pw.get("properties") or {}) - expr = str(pw.get("expr") or pw.get("expression") or "") - strat = str(pw.get("strategy") or "playwright") - c = LocatorCandidate( - strategy=strat, - scope=scope, - properties=props or {"expression": expr}, - expression=expr, - stability_score=float(pw.get("score") or 0.7), - uses_index=bool(pw.get("uses_index")), - uses_coordinates=bool(pw.get("uses_coordinates")), - reasons=["from_playwright_candidates"], - ) - if rows and props: - c.mark_counts(count_matches(rows, properties=props, scope=scope)) - c.validation = "snapshot_only" - cands.append(c) - - def add(strategy: str, props: dict[str, Any], score: float, reason: str) -> None: - if not props or not any(str(v).strip() for v in props.values()): - return - c = LocatorCandidate( - strategy=strategy, - scope=dict(scope), - properties=dict(props), - expression=_expr(strategy, props), - stability_score=score, - reasons=[reason], - ) - if rows: - c.mark_counts(count_matches(rows, properties=props, scope=scope)) - c.validation = "snapshot_only" - cands.append(c) - - if role and name: - add("role", {"role": role, "name": name}, 0.95, "role_name") - elif role and is_editable: - add("role", {"role": role}, 0.78, "editable_role") - if semantic_role and semantic_name: - add( - "semantic_role", - {"role": semantic_role, "name": semantic_name}, - 0.86, - "semantic_target_inferred_from_form_scope", - ) - if test_id: - add("test_id", {"test_id": test_id}, 0.93, "test_id") - if label: - add("label", {"label": label}, 0.88, "label") - if placeholder: - add("placeholder", {"placeholder": placeholder}, 0.84, "placeholder") - if name_attr: - add("name_attr", {"name_attr": name_attr}, 0.89, "html_name_attribute") - if href and role.lower() == "link": - add("href", {"href": href}, 0.91, "link_destination") - if text and len(text) <= 80: - add("text", {"text": text}, 0.75, "visible_text") - if css_id and not css_id[0].isdigit(): - add("css_id", {"id": css_id}, 0.8, "css_id") - if stable_css: - c = LocatorCandidate( - strategy="stable_css", - scope=dict(scope), - properties={"stable_css": stable_css}, - expression=f"css:{stable_css}", - stability_score=0.78 if ("#" in stable_css or "data-testid" in stable_css) else 0.62, - reasons=["stable_attribute_selector", "live_unique_required"], - ) - c.production_eligible = False - cands.append(c) - if css_path: - # Full/scoped CSS is a last-resort executable locator. It only becomes - # production eligible after live validation proves it is unique. - c = LocatorCandidate( - strategy="css", - scope=dict(scope), - properties={"css": css_path}, - expression=f"css:{css_path}", - stability_score=0.48 if "#" in css_path else 0.32, - reasons=["event_dom_path", "live_unique_required"], - ) - c.production_eligible = False - cands.append(c) - - # Never add coordinate locator as candidate for production - if el.get("bounds") or el.get("rect"): - c = LocatorCandidate( - strategy="bounding_rect", - scope=dict(scope), - properties={"bounds": el.get("bounds") or el.get("rect")}, - expression="coordinates:(evidence_only)", - stability_score=0.05, - uses_coordinates=True, - production_eligible=False, - reasons=["coordinates_evidence_only"], - ) - c.unique = False - c.ambiguous = True - cands.append(c) - - # Re-validate production_eligible after counts - for c in cands: - if c.actual_count is not None: - c.mark_counts(int(c.actual_count), expected=int(c.expected_count or 1)) - elif c.uses_coordinates: - c.production_eligible = False - - primary, _ = rank_and_select_primary(cands) - # Ensure ambiguous never becomes primary via side effect — rank handles it. - del primary - return cands diff --git a/tools/ui_inspector/locators/catalog.py b/tools/ui_inspector/locators/catalog.py deleted file mode 100644 index c8f7731..0000000 --- a/tools/ui_inspector/locators/catalog.py +++ /dev/null @@ -1,294 +0,0 @@ -"""Element Catalog: merge observations, locator history, parameterization.""" - -from __future__ import annotations - -import re -from pathlib import Path -from typing import Any, Optional - -from models.element import ElementCatalog, ElementRef, element_fingerprint -from models.locator import LocatorCandidate -from models.serialize import to_dict, write_model_json -from redaction import redact_element_fields -from core.policies.element_identity import element_identity_key - -from .base import rank_and_select_primary -from .browser import compile_browser_locators -from .validator import validate_locator_against_rows -from .windows import compile_windows_locators - -_DYNAMIC_VALUE_HINTS = re.compile( - r"(联系人|搜索|search|contact|user|chat|title|name)", - re.I, -) - - -def suggest_logical_name(element: dict[str, Any]) -> str: - aid = str(element.get("automation_id") or "").strip() - name = str(element.get("name") or "").strip() - role = str(element.get("control_type") or element.get("role") or "el").strip() - text = str(element.get("text") or "").strip() - semantic_name = str(element.get("semantic_name") or "").strip() - if element.get("contenteditable"): - base = aid or str(element.get("placeholder") or element.get("label") or "") or role - else: - base = aid or semantic_name or name or (text if 0 < len(text) <= 48 else "") - if not base: - for ancestor in element.get("parent_chain") or []: - ancestor_id = str((ancestor or {}).get("id") or "").strip() - ancestor_text = str((ancestor or {}).get("text") or "").strip() - if ancestor_id: - base = f"{ancestor_id}_{role}" - break - if ancestor_text and len(ancestor_text) <= 32: - base = f"{ancestor_text}_{role}" - break - base = base or role - base = re.sub(r"[^\w\u4e00-\u9fff]+", "_", base, flags=re.UNICODE).strip("_") - base = base[:48] or "element" - return base.lower() if base.isascii() else base - - -def _maybe_parameterize(value: str, *, field: str) -> tuple[str, dict[str, str]]: - """Replace likely dynamic business values with ${param} placeholders.""" - v = (value or "").strip() - if not v or len(v) > 64: - return v, {} - if field in {"value", "text", "name"} and _DYNAMIC_VALUE_HINTS.search(field): - return "${value}", {"value": v} - # Contact-like short Chinese/ASCII names in value when name is generic - if field == "value" and 1 < len(v) <= 32 and not v.startswith("${"): - # Only parameterize when looks like a person/search term (letters/CJK, not chrome) - if re.fullmatch(r"[\w\u4e00-\u9fff \-_.@]+", v, flags=re.UNICODE): - chrome = {"关闭", "最小化", "最大化", "取消", "确定", "发送", "搜索"} - if v not in chrome: - return "${contact_name}", {"contact_name": v} - return v, {} - - -class ElementCatalogService: - def __init__(self, flow_dir: Path, *, flow_id: str = "") -> None: - self.flow_dir = Path(flow_dir) - self.elements_dir = self.flow_dir / "elements" - self.elements_dir.mkdir(parents=True, exist_ok=True) - self.catalog_path = self.flow_dir / "element_catalog.json" - self.catalog = ElementCatalog(flow_id=flow_id) - self._by_fp: dict[str, ElementRef] = {} - self._by_identity: dict[str, ElementRef] = {} - self._by_name: dict[str, ElementRef] = {} - - def load(self) -> None: - if not self.catalog_path.exists(): - return - import json - from models.serialize import from_dict - - data = json.loads(self.catalog_path.read_text(encoding="utf-8")) - self.catalog.flow_id = data.get("flow_id") or self.catalog.flow_id - for item in data.get("elements") or []: - # Reconstruct LocatorCandidate lists lightly - locs = [] - for lc in item.get("locator_candidates") or []: - locs.append(from_dict(LocatorCandidate, lc)) - primary = None - if item.get("primary_locator"): - primary = from_dict(LocatorCandidate, item["primary_locator"]) - fallbacks = [ - from_dict(LocatorCandidate, x) for x in (item.get("fallback_locators") or []) - ] - el = ElementRef( - id=item.get("id") or "", - logical_name=item.get("logical_name") or "", - surface=item.get("surface") or "windows", - backend=item.get("backend") or "", - app=item.get("app") or {}, - window_scope=item.get("window_scope") or {}, - element_scope=item.get("element_scope") or {}, - role=item.get("role") or "", - control_type=item.get("control_type") or "", - name=item.get("name") or "", - value=item.get("value") or "", - automation_id=item.get("automation_id") or "", - class_name=item.get("class_name") or "", - framework_id=item.get("framework_id") or "", - states=item.get("states") or {}, - bounds=item.get("bounds") or {}, - actions=list(item.get("actions") or []), - parent_chain=list(item.get("parent_chain") or []), - raw=item.get("raw") or {}, - fingerprint=item.get("fingerprint") or "", - sensitive=bool(item.get("sensitive")), - locator_candidates=locs, - primary_locator=primary, - fallback_locators=fallbacks, - expected_count=int(item.get("expected_count") or 1), - verified_count=item.get("verified_count"), - verification_level=item.get("verification_level") or "none", - stability_score=float(item.get("stability_score") or 0.0), - verified_at=item.get("verified_at"), - first_seen=item.get("first_seen") or "", - last_seen=item.get("last_seen") or "", - observation_count=int(item.get("observation_count") or 1), - locator_history=list(item.get("locator_history") or []), - parameterized=dict(item.get("parameterized") or {}), - ) - if el.fingerprint: - self._by_fp[el.fingerprint] = el - identity = str((el.raw or {}).get("_identity_key") or "") - if not identity: - identity = element_identity_key( - el.raw or to_dict(el), - surface=el.surface, - page_url=str((el.element_scope or {}).get("page_url") or ""), - frame_name=str((el.element_scope or {}).get("frame_name") or ""), - ) - self._by_identity[identity] = el - if el.logical_name: - self._by_name[el.logical_name] = el - self.catalog.elements.append(el) - - def save(self) -> None: - from datetime import datetime - - self.catalog.updated_at = datetime.now().astimezone().isoformat() - write_model_json(self.catalog_path, self.catalog) - for el in self.catalog.elements: - write_model_json(self.elements_dir / f"{el.id}.json", el) - - def upsert_from_snapshot( - self, - element: dict[str, Any], - *, - surface: str, - backend: str = "", - window_scope: Optional[dict[str, Any]] = None, - rows: Optional[list[dict[str, Any]]] = None, - logical_name: Optional[str] = None, - frame: Optional[dict[str, Any]] = None, - live_counts: Optional[dict[str, int]] = None, - ) -> ElementRef: - cleaned, sensitive = redact_element_fields(dict(element or {})) - win = dict(window_scope or {}) - value = str(cleaned.get("value") or "") - is_editable = bool(cleaned.get("contenteditable")) - param_value, params = _maybe_parameterize(value, field="value") - if params: - cleaned["value"] = param_value - - fp = element_fingerprint( - surface=surface, - role=str(cleaned.get("control_type") or cleaned.get("role") or ""), - name=str(cleaned.get("name") or ""), - automation_id=str(cleaned.get("automation_id") or ""), - class_name=str(cleaned.get("class_name") or ""), - value=str(cleaned.get("value") or ""), - window_scope=win, - dom_fingerprint=str(cleaned.get("fingerprint") or cleaned.get("css") or ""), - page_scope=str((frame or {}).get("url") or win.get("url") or ""), - contenteditable=is_editable, - ) - - identity = element_identity_key( - cleaned, - surface=surface, - page_url=str((frame or {}).get("url") or win.get("url") or ""), - frame_name=str((frame or {}).get("name") or ""), - ) - cleaned["_identity_key"] = identity - - existing = self._by_identity.get(identity) or self._by_fp.get(fp) - if existing is None and logical_name and logical_name in self._by_name: - existing = self._by_name[logical_name] - - if surface == "browser": - cands = compile_browser_locators(cleaned, rows=rows, frame=frame) - else: - cands = compile_windows_locators(cleaned, rows=rows, window_scope=win) - - for c in cands: - if rows is not None and not c.uses_coordinates: - validate_locator_against_rows(c, rows, level="snapshot_only") - if live_counts is not None and c.strategy in live_counts: - c.mark_counts(int(live_counts[c.strategy])) - c.validation = "live" - - primary, fallbacks = rank_and_select_primary(cands) - # Ambiguous / coordinates never primary - if primary is not None and (primary.ambiguous or not primary.production_eligible): - primary = None - - lname = logical_name or (existing.logical_name if existing else "") or suggest_logical_name( - cleaned - ) - - if existing: - # Merge observation; keep locator history - if primary is not None: - old = existing.primary_locator - if old is not None and to_dict(old) != to_dict(primary): - existing.locator_history.append( - {"replaced_at": existing.last_seen, "locator": to_dict(old)} - ) - existing.primary_locator = primary - existing.locator_candidates = cands - existing.fallback_locators = fallbacks - existing.touch() - existing.raw = cleaned - existing.bounds = dict(cleaned.get("bounds") or cleaned.get("rect") or existing.bounds or {}) - if params: - existing.parameterized.update(params) - existing.sensitive = existing.sensitive or sensitive - existing.verification_level = "live" if live_counts else "snapshot_only" - self._by_fp[fp] = existing - self._by_identity[identity] = existing - if primary and primary.actual_count is not None: - existing.verified_count = primary.actual_count - self.save() - return existing - - el = ElementRef( - logical_name=lname, - surface=surface, - backend=backend, - window_scope=win, - element_scope={ - "page_url": str((frame or {}).get("url") or win.get("url") or ""), - "frame_name": str((frame or {}).get("name") or ""), - }, - role=str(cleaned.get("role") or ""), - control_type=str(cleaned.get("control_type") or ""), - name=str(cleaned.get("name") or ""), - value=str(cleaned.get("value") or ""), - automation_id=str(cleaned.get("automation_id") or ""), - class_name=str(cleaned.get("class_name") or ""), - framework_id=str(cleaned.get("framework_id") or ""), - states={ - "is_enabled": cleaned.get("is_enabled"), - "is_offscreen": cleaned.get("is_offscreen"), - }, - bounds=dict(cleaned.get("bounds") or cleaned.get("rect") or {}), - parent_chain=list(cleaned.get("parent_chain") or []), - raw=cleaned, - fingerprint=fp, - sensitive=sensitive, - locator_candidates=cands, - primary_locator=primary, - fallback_locators=fallbacks, - verification_level="live" if live_counts else "snapshot_only", - verified_count=(primary.actual_count if primary else None), - parameterized=params, - stability_score=float(primary.stability_score) if primary else 0.0, - ) - # Ensure unique logical_name - base = el.logical_name - i = 2 - while el.logical_name in self._by_name: - el.logical_name = f"{base}_{i}" - i += 1 - - self.catalog.elements.append(el) - self._by_fp[fp] = el - self._by_identity[identity] = el - self._by_name[el.logical_name] = el - self.save() - return el diff --git a/tools/ui_inspector/locators/validator.py b/tools/ui_inspector/locators/validator.py deleted file mode 100644 index a351e98..0000000 --- a/tools/ui_inspector/locators/validator.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Locator uniqueness validation against snapshot / live tree rows.""" - -from __future__ import annotations - -from typing import Any, Optional - -from models.locator import LocatorCandidate - -from .base import count_matches - - -def validate_locator_against_rows( - candidate: LocatorCandidate, - rows: list[dict[str, Any]], - *, - level: str = "snapshot_only", - expected_count: int = 1, -) -> LocatorCandidate: - """ - Validate structured locator properties against a row list. - - Levels supported in this phase: snapshot_only | live_tree - """ - actual = count_matches( - rows, - properties=candidate.properties, - scope=candidate.scope, - ) - candidate.mark_counts(actual, expected=expected_count) - candidate.validation = level - if not candidate.unique: - candidate.reasons = list(candidate.reasons or []) - if "ambiguous_match_count" not in candidate.reasons: - candidate.reasons.append("ambiguous_match_count") - candidate.production_eligible = False - if candidate.uses_coordinates: - candidate.production_eligible = False - if "coordinates_never_primary" not in (candidate.reasons or []): - candidate.reasons = list(candidate.reasons or []) + ["coordinates_never_primary"] - return candidate - - -def choose_production_primary( - candidates: list[LocatorCandidate], -) -> Optional[LocatorCandidate]: - """Return first production-eligible unique non-coordinate locator, else None.""" - for c in candidates: - if c.production_eligible and c.unique and not c.uses_coordinates: - return c - return None diff --git a/tools/ui_inspector/locators/windows.py b/tools/ui_inspector/locators/windows.py deleted file mode 100644 index cc591c4..0000000 --- a/tools/ui_inspector/locators/windows.py +++ /dev/null @@ -1,180 +0,0 @@ -"""Windows UIA locator compiler (Name ≠ Value; no coordinate primary).""" - -from __future__ import annotations - -from typing import Any, Optional - -from models.locator import LocatorCandidate - -from .base import count_matches, rank_and_select_primary - - -def _expr(strategy: str, props: dict[str, Any]) -> str: - parts = [f"{k}={v!r}" for k, v in props.items() if v not in (None, "")] - return f"{strategy}:" + ",".join(parts) - - -def compile_windows_locators( - element: dict[str, Any], - *, - rows: Optional[list[dict[str, Any]]] = None, - window_scope: Optional[dict[str, Any]] = None, -) -> list[LocatorCandidate]: - """ - Priority: - 1. Window scope + AutomationId + ControlType - 2. Window scope + stable Name + ControlType - 3. Window scope + Name + Value + ControlType - 4. Business parent + child attrs - 5. Visible Value + ControlType - 6. Relative ancestor scope - 7. Index with business reason (low score) - 8. BoundingRectangle evidence only - """ - el = element or {} - rows = list(rows or []) - win = dict(window_scope or el.get("window_scope") or {}) - scope = { - "window_class": win.get("class_name") or win.get("window_class") or "", - "window_title": win.get("title") or win.get("window_title") or "", - "process_name": win.get("process_name") or "", - } - cands: list[LocatorCandidate] = [] - - aid = str(el.get("automation_id") or "").strip() - name = str(el.get("name") or "").strip() - value = str(el.get("value") or "").strip() - ctype = str(el.get("control_type") or el.get("role") or "").strip() - class_name = str(el.get("class_name") or "").strip() - parent_chain = list(el.get("parent_chain") or []) - - def add( - strategy: str, - props: dict[str, Any], - score: float, - reason: str, - *, - uses_index: bool = False, - uses_coordinates: bool = False, - ) -> None: - if not props or not any(str(v).strip() for v in props.values() if not isinstance(v, (dict, list))): - return - c = LocatorCandidate( - strategy=strategy, - scope=dict(scope), - properties=dict(props), - expression=_expr(strategy, props), - stability_score=score, - uses_index=uses_index, - uses_coordinates=uses_coordinates, - reasons=[reason], - ) - if rows: - c.mark_counts(count_matches(rows, properties=props, scope=scope)) - c.validation = "snapshot_only" - elif uses_coordinates: - c.production_eligible = False - c.unique = False - c.ambiguous = True - cands.append(c) - - if aid and ctype: - add( - "window_automation_id_type", - {"automation_id": aid, "control_type": ctype}, - 0.96, - "automation_id_control_type", - ) - elif aid: - add("window_automation_id", {"automation_id": aid}, 0.9, "automation_id") - - if name and ctype and name.lower() not in {"item", "pane", "unknown"}: - add( - "window_name_type", - {"name": name, "control_type": ctype}, - 0.88, - "stable_name_control_type", - ) - - if name and value and ctype: - add( - "window_name_value_type", - {"name": name, "value": value, "control_type": ctype}, - 0.86, - "name_value_control_type", - ) - - # Business parent + child - if parent_chain: - parent = parent_chain[0] if isinstance(parent_chain[0], dict) else {} - parent_name = str(parent.get("name") or parent.get("automation_id") or "").strip() - if parent_name and (aid or name or value): - props: dict[str, Any] = {"parent_name": parent_name} - if aid: - props["automation_id"] = aid - if name: - props["name"] = name - if value: - props["value"] = value - if ctype: - props["control_type"] = ctype - add("parent_scoped", props, 0.82, "business_parent_scope") - - if value and ctype and len(value) <= 64: - add( - "value_type", - {"value": value, "control_type": ctype}, - 0.78, - "visible_value_control_type", - ) - - if class_name and ctype: - add( - "class_type", - {"class_name": class_name, "control_type": ctype}, - 0.55, - "class_name_control_type", - ) - - # Index only with explicit business reason — low score, not auto primary - idx = el.get("index") - if idx is not None and (aid or name): - props = {"index": int(idx)} - if aid: - props["automation_id"] = aid - if name: - props["name"] = name - if ctype: - props["control_type"] = ctype - add( - "indexed", - props, - 0.35, - "index_requires_business_justification", - uses_index=True, - ) - - bounds = el.get("bounds") or el.get("rect") or el.get("bounding_rectangle") - if bounds: - c = LocatorCandidate( - strategy="bounding_rectangle", - scope=dict(scope), - properties={"bounds": bounds}, - expression="coordinates:(evidence_only)", - stability_score=0.05, - uses_coordinates=True, - production_eligible=False, - unique=False, - ambiguous=True, - reasons=["coordinates_never_primary"], - ) - cands.append(c) - - for c in cands: - if c.actual_count is not None: - c.mark_counts(int(c.actual_count), expected=int(c.expected_count or 1)) - if c.uses_coordinates: - c.production_eligible = False - - rank_and_select_primary(cands) - return cands diff --git a/tools/ui_inspector/models/__init__.py b/tools/ui_inspector/models/__init__.py deleted file mode 100644 index 2aa20b5..0000000 --- a/tools/ui_inspector/models/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -"""UI Inspector v2 data models (Action / State / Element / Flow).""" - -from .action import ACTION_TYPES, ActionFlow, ActionRecord -from .element import ElementRef -from .locator import LocatorCandidate -from .observation import UIObservation -from .serialize import from_dict, to_dict, write_model_json -from .state import UIState, compute_state_fingerprint - -__all__ = [ - "ACTION_TYPES", - "ActionFlow", - "ActionRecord", - "ElementRef", - "LocatorCandidate", - "UIState", - "UIObservation", - "compute_state_fingerprint", - "from_dict", - "to_dict", - "write_model_json", -] diff --git a/tools/ui_inspector/models/action.py b/tools/ui_inspector/models/action.py deleted file mode 100644 index 9a88597..0000000 --- a/tools/ui_inspector/models/action.py +++ /dev/null @@ -1,119 +0,0 @@ -"""ActionRecord + ActionFlow models.""" - -from __future__ import annotations - -import uuid -from dataclasses import dataclass, field -from datetime import datetime -from typing import Any, Optional - -SCHEMA_ACTION = "openclaw.ui_inspector.action.v1" -SCHEMA_ACTION_FLOW = "openclaw.ui_inspector.action_flow.v1" - -# Full vocabulary — capability flags live elsewhere for unimplemented types. -ACTION_TYPES: tuple[str, ...] = ( - "click", - "double_click", - "right_click", - "fill", - "type", - "clear", - "press_key", - "hotkey", - "select", - "toggle", - "hover", - "scroll", - "drag_drop", - "focus", - "window_switch", - "navigation", - "checkpoint", - "wait", - "verify", - "unknown", -) - -# Capability status for this Inspector release. -ACTION_CAPABILITY: dict[str, str] = { - "click": "implemented", - "double_click": "schema_only", - "right_click": "schema_only", - "fill": "implemented", - "type": "implemented", # aggregated into fill/type - "clear": "schema_only", - "press_key": "implemented", - "hotkey": "schema_only", - "select": "schema_only", - "toggle": "schema_only", - "hover": "schema_only", - "scroll": "schema_only", - "drag_drop": "schema_only", - "focus": "schema_only", - "window_switch": "schema_only", - "navigation": "partial", - "checkpoint": "implemented", - "wait": "schema_only", - "verify": "schema_only", - "unknown": "fallback", -} - - -def _now_iso() -> str: - return datetime.now().astimezone().isoformat() - - -def _new_id(prefix: str = "act") -> str: - return f"{prefix}_{uuid.uuid4().hex[:12]}" - - -@dataclass -class ActionRecord: - schema: str = SCHEMA_ACTION - id: str = field(default_factory=lambda: _new_id("act")) - sequence: int = 0 - action_type: str = "unknown" - started_at: str = field(default_factory=_now_iso) - ended_at: Optional[str] = None - surface: str = "windows" - target_element_id: Optional[str] = None - target_snapshot: dict[str, Any] = field(default_factory=dict) - input: dict[str, Any] = field(default_factory=dict) - sensitive: bool = False - before_state_id: Optional[str] = None - after_state_id: Optional[str] = None - before_observation_id: Optional[str] = None - after_observation_id: Optional[str] = None - observed_changes: list[dict[str, Any]] = field(default_factory=list) - proposed_postconditions: list[dict[str, Any]] = field(default_factory=list) - status: str = "ok" # ok | partial | failed | skipped - confidence: float = 0.0 - evidence: dict[str, Any] = field(default_factory=dict) - raw_event: dict[str, Any] = field(default_factory=dict) - raw_event_ids: list[str] = field(default_factory=list) - capability: str = "" - - def __post_init__(self) -> None: - if self.action_type not in ACTION_TYPES: - self.action_type = "unknown" - if not self.capability: - self.capability = ACTION_CAPABILITY.get(self.action_type, "schema_only") - - -@dataclass -class ActionFlow: - schema: str = SCHEMA_ACTION_FLOW - flow_id: str = "" - flow_name: str = "" - created_at: str = field(default_factory=_now_iso) - ended_at: Optional[str] = None - states: list[str] = field(default_factory=list) # state ids in order seen - observations: list[str] = field(default_factory=list) # every concrete capture - actions: list[str] = field(default_factory=list) # action ids in sequence - elements: list[str] = field(default_factory=list) # element ids - entry_state_id: Optional[str] = None - final_state_id: Optional[str] = None - metadata: dict[str, Any] = field(default_factory=dict) - recording_mode: str = "auto" # auto | guided | checkpoint | legacy_interval - surface: str = "" - status: str = "open" diff --git a/tools/ui_inspector/models/element.py b/tools/ui_inspector/models/element.py deleted file mode 100644 index 757e562..0000000 --- a/tools/ui_inspector/models/element.py +++ /dev/null @@ -1,126 +0,0 @@ -"""ElementRef model for Element Catalog.""" - -from __future__ import annotations - -import hashlib -import json -import uuid -from dataclasses import dataclass, field -from datetime import datetime -from typing import Any, Optional - -from .locator import LocatorCandidate - -SCHEMA_ELEMENT = "openclaw.ui_inspector.element.v1" -SCHEMA_CATALOG = "openclaw.ui_inspector.element_catalog.v1" - - -def _now_iso() -> str: - return datetime.now().astimezone().isoformat() - - -def _new_id(prefix: str = "el") -> str: - return f"{prefix}_{uuid.uuid4().hex[:12]}" - - -def element_fingerprint( - *, - surface: str, - role: str = "", - name: str = "", - automation_id: str = "", - class_name: str = "", - value: str = "", - window_scope: Optional[dict[str, Any]] = None, - dom_fingerprint: str = "", - page_scope: str = "", - contenteditable: bool = False, -) -> str: - payload = { - "surface": surface or "", - "role": role or "", - "name": "" if contenteditable else (name or ""), - "automation_id": automation_id or "", - "class_name": class_name or "", - # Value is identity only when short/stable-looking (not long free text). - "value": "" if contenteditable else ((value or "") if len(value or "") <= 64 else ""), - "dom_fingerprint": dom_fingerprint or "", - "page_scope": page_scope or "", - "window_scope": { - "class_name": (window_scope or {}).get("class_name") or "", - "title": (window_scope or {}).get("title") or "", - }, - } - raw = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) - return hashlib.sha256(raw.encode("utf-8")).hexdigest() - - -@dataclass -class ElementRef: - schema: str = SCHEMA_ELEMENT - id: str = field(default_factory=lambda: _new_id("el")) - logical_name: str = "" - surface: str = "windows" - backend: str = "" - app: dict[str, Any] = field(default_factory=dict) - window_scope: dict[str, Any] = field(default_factory=dict) - element_scope: dict[str, Any] = field(default_factory=dict) - role: str = "" - control_type: str = "" - name: str = "" - value: str = "" - automation_id: str = "" - class_name: str = "" - framework_id: str = "" - states: dict[str, Any] = field(default_factory=dict) - bounds: dict[str, Any] = field(default_factory=dict) - actions: list[str] = field(default_factory=list) - parent_chain: list[dict[str, Any]] = field(default_factory=list) - raw: dict[str, Any] = field(default_factory=dict) - fingerprint: str = "" - sensitive: bool = False - locator_candidates: list[LocatorCandidate] = field(default_factory=list) - primary_locator: Optional[LocatorCandidate] = None - fallback_locators: list[LocatorCandidate] = field(default_factory=list) - expected_count: int = 1 - verified_count: Optional[int] = None - verification_level: str = "none" - stability_score: float = 0.0 - verified_at: Optional[str] = None - first_seen: str = field(default_factory=_now_iso) - last_seen: str = field(default_factory=_now_iso) - observation_count: int = 1 - locator_history: list[dict[str, Any]] = field(default_factory=list) - parameterized: dict[str, str] = field(default_factory=dict) - - def ensure_fingerprint(self) -> str: - if self.fingerprint: - return self.fingerprint - self.fingerprint = element_fingerprint( - surface=self.surface, - role=self.role or self.control_type, - name=self.name, - automation_id=self.automation_id, - class_name=self.class_name, - value=self.value, - window_scope=self.window_scope, - dom_fingerprint=str(self.raw.get("fingerprint") or self.raw.get("css") or ""), - page_scope=str(self.element_scope.get("page_url") or ""), - contenteditable=bool(self.raw.get("contenteditable")), - ) - return self.fingerprint - - def touch(self) -> None: - self.last_seen = _now_iso() - self.observation_count = int(self.observation_count or 0) + 1 - - -@dataclass -class ElementCatalog: - schema: str = SCHEMA_CATALOG - flow_id: str = "" - updated_at: str = field(default_factory=_now_iso) - elements: list[ElementRef] = field(default_factory=list) - - def by_logical_name(self) -> dict[str, ElementRef]: - return {e.logical_name: e for e in self.elements if e.logical_name} diff --git a/tools/ui_inspector/models/flow.py b/tools/ui_inspector/models/flow.py deleted file mode 100644 index 8883ca7..0000000 --- a/tools/ui_inspector/models/flow.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Flow-level helpers (re-export ActionFlow for models.flow import path).""" - -from .action import ActionFlow, ActionRecord -from .element import ElementCatalog, ElementRef -from .state import UIState -from .observation import UIObservation - -__all__ = [ - "ActionFlow", "ActionRecord", "ElementCatalog", "ElementRef", "UIState", - "UIObservation", -] diff --git a/tools/ui_inspector/models/locator.py b/tools/ui_inspector/models/locator.py deleted file mode 100644 index d0c5864..0000000 --- a/tools/ui_inspector/models/locator.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Structured locator candidate model.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Any, Optional - - -SCHEMA_LOCATOR = "openclaw.ui_inspector.locator_candidate.v1" - - -@dataclass -class LocatorCandidate: - schema: str = SCHEMA_LOCATOR - strategy: str = "" - scope: dict[str, Any] = field(default_factory=dict) - properties: dict[str, Any] = field(default_factory=dict) - expression: str = "" - expected_count: int = 1 - actual_count: Optional[int] = None - unique: Optional[bool] = None - stability_score: float = 0.0 - uses_index: bool = False - uses_coordinates: bool = False - validation: str = "none" # snapshot_only | live_tree | cross_state | cross_session | none - reasons: list[str] = field(default_factory=list) - ambiguous: bool = False - production_eligible: bool = False - - def mark_counts(self, actual: int, *, expected: int = 1) -> None: - self.expected_count = int(expected) - self.actual_count = int(actual) - self.unique = self.actual_count == self.expected_count - self.ambiguous = not bool(self.unique) - # Coordinates / index never become production primary automatically. - self.production_eligible = bool( - self.unique - and not self.uses_coordinates - and not (self.uses_index and self.stability_score < 0.7) - ) diff --git a/tools/ui_inspector/models/observation.py b/tools/ui_inspector/models/observation.py deleted file mode 100644 index 8db5713..0000000 --- a/tools/ui_inspector/models/observation.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Per-capture evidence model. - -UIState represents a deduplicated semantic state. UIObservation represents -one concrete capture and is therefore never deduplicated. -""" - -from __future__ import annotations - -import uuid -from dataclasses import dataclass, field -from datetime import datetime -from typing import Any, Optional - -SCHEMA_OBSERVATION = "openclaw.ui_inspector.observation.v1" - - -def _now_iso() -> str: - return datetime.now().astimezone().isoformat() - - -def _new_id() -> str: - return f"obs_{uuid.uuid4().hex[:12]}" - - -@dataclass -class UIObservation: - schema: str = SCHEMA_OBSERVATION - id: str = field(default_factory=_new_id) - captured_at: str = field(default_factory=_now_iso) - state_id: str = "" - previous_observation_id: Optional[str] = None - change_reason: str = "capture" - surface: str = "windows" - driver: str = "" - page_id: str = "" - document_id: str = "" - frame_id: str = "" - url: str = "" - page_title: str = "" - tree_path: str = "" - index_path: str = "" - locatable_path: str = "" - screenshot_path: str = "" - stats: dict[str, Any] = field(default_factory=dict) - selection: dict[str, Any] = field(default_factory=dict) - may_contain_sensitive_data: bool = False - diff --git a/tools/ui_inspector/models/serialize.py b/tools/ui_inspector/models/serialize.py deleted file mode 100644 index f566a8e..0000000 --- a/tools/ui_inspector/models/serialize.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Stable JSON serialization helpers for v2 models.""" - -from __future__ import annotations - -import json -from dataclasses import asdict, is_dataclass -from datetime import datetime -from pathlib import Path -from typing import Any, TypeVar - -T = TypeVar("T") - - -def _normalize(obj: Any) -> Any: - if obj is None or isinstance(obj, (str, int, float, bool)): - return obj - if isinstance(obj, datetime): - return obj.isoformat() - if isinstance(obj, Path): - return str(obj).replace("\\", "/") - if is_dataclass(obj) and not isinstance(obj, type): - return {k: _normalize(v) for k, v in asdict(obj).items()} - if isinstance(obj, dict): - return {str(k): _normalize(v) for k, v in obj.items()} - if isinstance(obj, (list, tuple)): - return [_normalize(x) for x in obj] - return str(obj) - - -def to_dict(obj: Any) -> dict[str, Any]: - data = _normalize(obj) - if not isinstance(data, dict): - raise TypeError(f"expected dataclass/dict, got {type(obj)!r}") - return data - - -def from_dict(cls: type[T], data: dict[str, Any]) -> T: - """Best-effort dataclass reconstruction (ignores unknown keys).""" - if not is_dataclass(cls): - raise TypeError(f"{cls!r} is not a dataclass") - import dataclasses - - fields = {f.name for f in dataclasses.fields(cls)} - kwargs = {k: v for k, v in (data or {}).items() if k in fields} - return cls(**kwargs) # type: ignore[call-arg] - - -def dumps_json(obj: Any, *, indent: int = 2) -> str: - return json.dumps( - to_dict(obj) if is_dataclass(obj) and not isinstance(obj, type) else _normalize(obj), - ensure_ascii=False, - indent=indent, - sort_keys=False, - ) + "\n" - - -def write_model_json(path: Path, obj: Any) -> None: - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(dumps_json(obj), encoding="utf-8") - - -def read_json(path: Path) -> Any: - return json.loads(Path(path).read_text(encoding="utf-8")) diff --git a/tools/ui_inspector/models/state.py b/tools/ui_inspector/models/state.py deleted file mode 100644 index c483fe0..0000000 --- a/tools/ui_inspector/models/state.py +++ /dev/null @@ -1,175 +0,0 @@ -"""UIState model + stable business fingerprint.""" - -from __future__ import annotations - -import hashlib -import json -import uuid -from dataclasses import dataclass, field -from datetime import datetime, timezone -from typing import Any, Optional - -SCHEMA_STATE = "openclaw.ui_inspector.state.v1" - -# Keys ignored when hashing business identity. -_FINGERPRINT_EXCLUDE_TOP = { - "captured_at", - "id", - "previous_state_id", - "tree_path", - "index_path", - "locatable_path", - "screenshot_path", - "duplicate_of", - "observation_count", - "change_reason", -} - -_SESSION_KEYS = { - "pid", - "hwnd", - "process_id", - "thread_id", - "elapsed_ms", - "dump_ms", - "warmup_ms", - "timings", - "screenshot", - "screen_path", -} - - -def _now_iso() -> str: - return datetime.now().astimezone().isoformat() - - -def _new_id(prefix: str = "st") -> str: - return f"{prefix}_{uuid.uuid4().hex[:12]}" - - -def _strip_session(obj: Any) -> Any: - if isinstance(obj, dict): - out: dict[str, Any] = {} - for k, v in obj.items(): - lk = str(k).lower() - if lk in _SESSION_KEYS or lk.endswith("_ms") or lk.endswith("_path"): - continue - if lk in {"left", "top", "right", "bottom", "x", "y", "width", "height"}: - # Absolute coordinates are not primary identity. - continue - if lk == "rect" or lk == "bounds" or lk == "bounding_rectangle": - continue - out[str(k)] = _strip_session(v) - return out - if isinstance(obj, list): - return [_strip_session(x) for x in obj] - return obj - - -def _locatable_signature(rows: list[dict[str, Any]], *, limit: int = 120) -> list[dict[str, Any]]: - """Compact visible-business signature for fingerprinting.""" - sig: list[dict[str, Any]] = [] - for r in rows[:limit]: - if not isinstance(r, dict): - continue - entry = { - "control_type": r.get("control_type") or r.get("role") or "", - "name": r.get("name") or "", - "value": r.get("value") or r.get("text") or "", - "automation_id": r.get("automation_id") or r.get("test_id") or "", - "class_name": r.get("class_name") or "", - } - # Drop empty noise rows - if not any(str(v).strip() for v in entry.values()): - continue - sig.append(entry) - # Stable order - sig.sort( - key=lambda e: ( - str(e.get("control_type") or ""), - str(e.get("automation_id") or ""), - str(e.get("name") or ""), - str(e.get("value") or ""), - ) - ) - return sig - - -def compute_state_fingerprint( - *, - surface: str, - driver: str = "", - url: str = "", - page_title: str = "", - window: Optional[dict[str, Any]] = None, - locatable: Optional[list[dict[str, Any]]] = None, - app: Optional[dict[str, Any]] = None, - extra_business: Optional[dict[str, Any]] = None, -) -> str: - """ - SHA-256 of stable business identity. - - Excludes timestamps, pid/hwnd, screenshot paths, absolute coords, timings. - """ - win = dict(window or {}) - app_d = dict(app or {}) - payload = { - "surface": surface or "", - "driver": driver or "", - "url": url or win.get("url") or app_d.get("url") or "", - "page_title": page_title or "", - "window": { - "title": win.get("title") or app_d.get("title") or "", - "class_name": win.get("class_name") or app_d.get("class_name") or "", - "process_name": win.get("process_name") or app_d.get("process_name") or "", - }, - "locatable": _locatable_signature(list(locatable or [])), - "extra": _strip_session(extra_business or {}), - } - raw = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) - return hashlib.sha256(raw.encode("utf-8")).hexdigest() - - -@dataclass -class UIState: - schema: str = SCHEMA_STATE - id: str = field(default_factory=lambda: _new_id("st")) - surface: str = "windows" - driver: str = "" - captured_at: str = field(default_factory=_now_iso) - fingerprint: str = "" - previous_state_id: Optional[str] = None - change_reason: str = "" - app: dict[str, Any] = field(default_factory=dict) - window: dict[str, Any] = field(default_factory=dict) - url: str = "" - page_title: str = "" - tree_path: str = "" - index_path: str = "" - locatable_path: str = "" - screenshot_path: str = "" - stats: dict[str, Any] = field(default_factory=dict) - selection: dict[str, Any] = field(default_factory=dict) - duplicate_of: Optional[str] = None - observation_count: int = 1 - may_contain_sensitive_data: bool = False - # In-memory locatable used for fingerprint/diff (not always persisted as field). - locatable: list[dict[str, Any]] = field(default_factory=list, repr=False) - - def ensure_fingerprint(self) -> str: - if self.fingerprint: - return self.fingerprint - self.fingerprint = compute_state_fingerprint( - surface=self.surface, - driver=self.driver, - url=self.url, - page_title=self.page_title, - window=self.window, - locatable=self.locatable, - app=self.app, - ) - return self.fingerprint - - def bump_observation(self) -> None: - self.observation_count = int(self.observation_count or 0) + 1 - self.captured_at = _now_iso() diff --git a/tools/ui_inspector/recorder/__init__.py b/tools/ui_inspector/recorder/__init__.py deleted file mode 100644 index 43b0d27..0000000 --- a/tools/ui_inspector/recorder/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Recorder package: observe user actions outside target processes.""" - -from .base import RawUserEvent, RecorderBase, RecorderState -from .input_aggregator import InputAggregator -from .manager import ActionRecordingManager - -__all__ = [ - "RawUserEvent", - "RecorderBase", - "RecorderState", - "InputAggregator", - "ActionRecordingManager", -] diff --git a/tools/ui_inspector/recorder/base.py b/tools/ui_inspector/recorder/base.py deleted file mode 100644 index af1300b..0000000 --- a/tools/ui_inspector/recorder/base.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Recorder abstractions.""" - -from __future__ import annotations - -import threading -from abc import ABC, abstractmethod -from dataclasses import dataclass, field -from datetime import datetime -from typing import Any, Callable, Optional - - -def _now_iso() -> str: - return datetime.now().astimezone().isoformat() - - -@dataclass -class RawUserEvent: - kind: str # mouse_down | mouse_up | key | input | navigation | focus | checkpoint - surface: str - timestamp: str = field(default_factory=_now_iso) - button: str = "" - key: str = "" - text: str = "" - x: Optional[int] = None - y: Optional[int] = None - target: dict[str, Any] = field(default_factory=dict) - frame: dict[str, Any] = field(default_factory=dict) - url: str = "" - meta: dict[str, Any] = field(default_factory=dict) - from_inspector: bool = False - - -class RecorderState: - IDLE = "idle" - RECORDING = "recording" - PAUSED = "paused" - STOPPED = "stopped" - - -class RecorderBase(ABC): - """Process-external observer. No injection / hooks into target.""" - - def __init__( - self, - *, - on_event: Optional[Callable[[RawUserEvent], None]] = None, - exclude_hwnds: Optional[Callable[[], set[int]]] = None, - ) -> None: - self.on_event = on_event - self.exclude_hwnds = exclude_hwnds or (lambda: set()) - self.status = RecorderState.IDLE - self._thread: Optional[threading.Thread] = None - self._stop = threading.Event() - - def emit(self, event: RawUserEvent) -> None: - if self.status != RecorderState.RECORDING: - return - if event.from_inspector: - return - if self.on_event: - self.on_event(event) - - @abstractmethod - def start(self) -> None: ... - - @abstractmethod - def stop(self) -> None: ... - - def pause(self) -> None: - if self.status == RecorderState.RECORDING: - self.status = RecorderState.PAUSED - - def resume(self) -> None: - if self.status == RecorderState.PAUSED: - self.status = RecorderState.RECORDING - - @property - def capability(self) -> dict[str, str]: - return {} diff --git a/tools/ui_inspector/recorder/browser.py b/tools/ui_inspector/recorder/browser.py deleted file mode 100644 index 5a3c098..0000000 --- a/tools/ui_inspector/recorder/browser.py +++ /dev/null @@ -1,740 +0,0 @@ -"""Browser recorder for Inspector-owned Playwright sessions. - -Injects capture-phase listeners that do NOT preventDefault / stopPropagation -and do NOT mutate business DOM. Re-attaches after navigation. -""" - -from __future__ import annotations - -import json -import queue -import threading -import time -from typing import Any, Callable, Optional - -from .base import RawUserEvent, RecorderBase, RecorderState - -_LISTEN_SCRIPT = r""" -(() => { - if (window.__openclawUiInspectorRecorderInstalled) return; - window.__openclawUiInspectorRecorderInstalled = true; - window.__openclawUiInspectorEvents = window.__openclawUiInspectorEvents || []; - window.__openclawUiInspectorDocId = window.__openclawUiInspectorDocId || - (Date.now().toString(36) + "-" + Math.random().toString(36).slice(2)); - window.__openclawUiInspectorSeq = window.__openclawUiInspectorSeq || 0; - - function cssPath(el) { - if (!el || !el.tagName) return ""; - const esc = (value) => { - try { return CSS.escape(String(value)); } catch (e) { return String(value).replace(/[^a-zA-Z0-9_-]/g, "\\$&"); } - }; - const parts = []; - let cur = el; - for (let i = 0; cur && i < 12; i++) { - let part = cur.tagName.toLowerCase(); - if (cur.id) { part += "#" + esc(cur.id); parts.unshift(part); break; } - const cls = (cur.className && typeof cur.className === "string") - ? cur.className.trim().split(/\s+/).slice(0, 3).map(esc).join(".") - : ""; - if (cls) part += "." + cls; - parts.unshift(part); - cur = cur.parentElement; - } - return parts.join(" > "); - } - - function stableCss(el) { - if (!el || !el.getAttribute) return ""; - const quote = (value) => JSON.stringify(String(value)); - const ownId = el.id || ""; - if (ownId) return "#" + CSS.escape(ownId); - const testId = el.getAttribute("data-testid") || ""; - if (testId) return `[data-testid=${quote(testId)}]`; - const tag = (el.tagName || "").toLowerCase(); - let leaf = ""; - const href = el.getAttribute("href") || ""; - if (tag === "a" && href) leaf = `a[href=${quote(href)}]`; - else if (el.getAttribute("contenteditable") != null) leaf = `${tag || "*"}[contenteditable]`; - else if (el.getAttribute("name")) leaf = `${tag || "*"}[name=${quote(el.getAttribute("name"))}]`; - else if (el.getAttribute("aria-label")) leaf = `${tag || "*"}[aria-label=${quote(el.getAttribute("aria-label"))}]`; - else if (el.getAttribute("placeholder")) leaf = `${tag || "*"}[placeholder=${quote(el.getAttribute("placeholder"))}]`; - const stableClasses = (node) => { - if (!node || typeof node.className !== "string") return []; - return node.className.trim().split(/\s+/).filter((token) => { - if (!token || token.length < 4 || token.length > 64) return false; - const lower = token.toLowerCase(); - const semantic = /(button|btn|input|editor|control|action|submit|send|publish|post|play|pause|menu|option|tab|dialog|form|toolbar|container)/.test(lower); - const structured = /[-_]/.test(token) && !/^[a-z0-9_-]{6,14}$/i.test(token); - return semantic || structured; - }).slice(0, 2); - }; - if (!leaf) { - const classes = stableClasses(el); - leaf = tag + classes.map(cls => "." + CSS.escape(cls)).join(""); - } - let cur = el.parentElement; - const relative = [leaf]; - for (let i = 0; cur && i < 10; i++, cur = cur.parentElement) { - if (cur.id) return `#${CSS.escape(cur.id)} > ${relative.reverse().join(" > ")}`; - const ancestorTestId = cur.getAttribute?.("data-testid") || ""; - if (ancestorTestId) return `[data-testid=${quote(ancestorTestId)}] > ${relative.reverse().join(" > ")}`; - const curTag = (cur.tagName || "").toLowerCase(); - const classes = stableClasses(cur); - relative.push(curTag + classes.map(cls => "." + CSS.escape(cls)).join("")); - } - return ""; - } - - function editableValue(el) { - if (!el) return ""; - if (el.value != null) return String(el.value); - if (el.isContentEditable || el.getAttribute?.("contenteditable") != null) { - return String(el.innerText != null ? el.innerText : (el.textContent || "")); - } - return ""; - } - - function isEditable(el) { - if (!el || !el.getAttribute) return false; - const tag = (el.tagName || "").toLowerCase(); - return tag === "input" || tag === "textarea" || tag === "select" || - !!el.isContentEditable || el.getAttribute("contenteditable") != null; - } - - function editableRoot(el) { - let cur = el; - for (let i = 0; cur && i < 12; i++) { - if (cur.getAttribute && cur.getAttribute("contenteditable") != null) return cur; - const tag = (cur.tagName || "").toLowerCase(); - if (tag === "input" || tag === "textarea" || tag === "select") return cur; - cur = cur.parentElement; - } - return null; - } - - function semanticRole(el) { - if (!el || !el.getAttribute) return ""; - const explicit = el.getAttribute("role"); - if (explicit) return explicit; - const tag = (el.tagName || "").toLowerCase(); - if (el.isContentEditable) return "textbox"; - if (tag === "a") return "link"; - if (tag === "textarea") return "textbox"; - if (tag === "select") return "combobox"; - if (tag === "input") { - const type = (el.type || "text").toLowerCase(); - if (type === "checkbox") return "checkbox"; - if (type === "radio") return "radio"; - if (["button", "submit", "reset"].includes(type)) return "button"; - return "textbox"; - } - return tag; - } - - function actionableFromEvent(ev) { - const path = (ev && typeof ev.composedPath === "function") ? ev.composedPath() : []; - const nodes = path.length ? path : [ev && ev.target]; - let best = ev && ev.target; - let bestScore = -1; - for (const el of nodes) { - if (!el || !el.matches) continue; - const tag = (el.tagName || "").toLowerCase(); - const role = (el.getAttribute("role") || "").toLowerCase(); - const cls = typeof el.className === "string" ? el.className.toLowerCase() : ""; - let cursor = ""; - try { cursor = getComputedStyle(el).cursor || ""; } catch (e) {} - let score = 0; - if (el.matches("button,a,input,textarea,select,summary,label")) score = 100; - if (el.isContentEditable || el.getAttribute("contenteditable") != null) score = 98; - if (["button","link","textbox","combobox","checkbox","radio","menuitem","option","tab"].includes(role)) score = Math.max(score, 96); - else if (role) score = Math.max(score, 52); - if (el.getAttribute("data-testid") || el.getAttribute("aria-label")) score = Math.max(score, 90); - if (el.getAttribute("tabindex") != null) score = Math.max(score, 74); - if (cursor === "pointer") score = Math.max(score, 68); - // Many production apps use camelCase classes such as sendControl or - // playButton. Prefer that actionable wrapper over its inherited-cursor - // SVG/path child while still retaining hit_target for diagnostics. - if (tag.includes("-") || /(button|btn|submit|send|play|control|action|click)/i.test(cls)) score = Math.max(score, 72); - if (score > bestScore) { best = el; bestScore = score; } - } - return best; - } - - function snap(el) { - if (!el || !el.getAttribute) return {}; - const role = semanticRole(el); - let cursor = ""; - let rect = {}; - try { - cursor = getComputedStyle(el).cursor || ""; - const r = el.getBoundingClientRect(); - rect = {left:r.left, top:r.top, right:r.right, bottom:r.bottom, width:r.width, height:r.height}; - } catch (e) {} - const tag = el.tagName ? el.tagName.toLowerCase() : ""; - const css = cssPath(el); - const stable = stableCss(el); - let cssCount = null; - let stableCssCount = null; - try { cssCount = css ? document.querySelectorAll(css).length : null; } catch (e) {} - try { stableCssCount = stable ? document.querySelectorAll(stable).length : null; } catch (e) {} - const href = el.getAttribute("href") || ""; - let hrefCount = null; - try { - hrefCount = href ? Array.from(document.querySelectorAll("a[href]")).filter( - node => node.getAttribute("href") === href - ).length : null; - } catch (e) {} - const roleNames = ["button","link","textbox","combobox","checkbox","radio","menuitem","option","tab"]; - return { - role: role, - explicit_role: el.getAttribute("role") || "", - aria_label: el.getAttribute("aria-label") || "", - name: el.getAttribute("aria-label") || el.name || el.id || - ((el.matches?.("button,a,[role],label,summary") || roleNames.includes(role) || cursor === "pointer" || tag.includes("-")) - ? (el.innerText || el.textContent || "").trim().slice(0, 120) : ""), - text: (el.innerText || el.textContent || "").trim().slice(0, 120), - value: editableValue(el).slice(0, 500), - test_id: el.getAttribute("data-testid") || "", - name_attr: el.getAttribute("name") || "", - href: href, - href_count: hrefCount, - title: el.getAttribute("title") || "", - alt: el.getAttribute("alt") || "", - id: el.id || "", - placeholder: el.getAttribute("placeholder") || el.getAttribute("aria-placeholder") || "", - label: "", - input_type: (el.type || "").toLowerCase(), - tag: tag, - class_name: (typeof el.className === "string" ? el.className : "").slice(0, 300), - cursor: cursor, - rect: rect, - disabled: !!el.disabled || el.getAttribute("aria-disabled") === "true", - contenteditable: !!el.isContentEditable || el.getAttribute("contenteditable") != null, - contenteditable_attr: el.getAttribute("contenteditable") != null, - tabindex: el.getAttribute("tabindex"), - css: css, - css_count: cssCount, - stable_css: stable, - stable_css_count: stableCssCount, - fingerprint: [ - el.tagName?.toLowerCase() || "", el.id || "", - el.getAttribute("data-testid") || "", el.getAttribute("name") || "", - css - ].join("|"), - }; - } - - function ancestry(el) { - const out = []; - let cur = el; - for (let i = 0; cur && i < 12; i++) { - out.push(snap(cur)); - cur = cur.parentElement; - } - return out; - } - - function push(kind, payload) { - try { - const item = Object.assign({ - event_id: window.__openclawUiInspectorDocId + ":" + (++window.__openclawUiInspectorSeq), - document_id: window.__openclawUiInspectorDocId, - sequence: window.__openclawUiInspectorSeq, - kind: kind, - ts: Date.now(), - url: location.href, - title: document.title, - frame_url: location.href, - frame_name: window.name || "", - }, payload || {}); - window.__openclawUiInspectorEvents.push(item); - if (typeof window.__openclawUiInspectorEmit === "function") { - try { window.__openclawUiInspectorEmit(item); } catch (e) {} - } - if (window.__openclawUiInspectorEvents.length > 200) { - window.__openclawUiInspectorEvents.splice(0, 50); - } - } catch (e) {} - } - - document.addEventListener("click", (ev) => { - const hit = ev.target; - const action = actionableFromEvent(ev) || hit; - push("click", { - target: snap(action), - hit_target: snap(hit), - action_target: snap(action), - ancestry: ancestry(hit), - candidate_chain: ancestry(hit), - button: ev.button, - x: ev.clientX, - y: ev.clientY - }); - }, true); - - document.addEventListener("pointerdown", (ev) => { - const hit = ev.target; - push("pointerdown", { - target: snap(hit), - hit_target: snap(hit), - candidate_chain: ancestry(hit), - button: ev.button, - pointer_id: ev.pointerId, - pointer_type: ev.pointerType || "", - x: ev.clientX, - y: ev.clientY - }); - }, true); - - document.addEventListener("input", (ev) => { - if (ev.isComposing) return; - const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target; - push("input", { - target: snap(t), text: editableValue(t), input_event: ev.inputType || "input", - candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted - }); - }, true); - - document.addEventListener("change", (ev) => { - const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target; - push("input", { target: snap(t), text: editableValue(t), input_event: "change", - candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted }); - }, true); - - document.addEventListener("compositionend", (ev) => { - const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target; - push("input", { - target: snap(t), text: editableValue(t), input_event: "compositionend", - candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted - }); - }, true); - - document.addEventListener("paste", (ev) => { - const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target; - setTimeout(() => push("input", { - target: snap(t), text: editableValue(t), input_event: "paste", - candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted - }), 0); - }, true); - - document.addEventListener("keydown", (ev) => { - const special = ["Enter","Escape","Tab","Backspace","Delete"]; - const editing = isEditable(ev.target); - const mutatingEditKey = editing && (ev.key === "Backspace" || ev.key === "Delete"); - if (!mutatingEditKey && (special.indexOf(ev.key) >= 0 || (ev.key && ev.key.length > 1))) { - const t = actionableFromEvent(ev) || ev.target; - push("key", { key: ev.key, target: snap(t) }); - } - }, true); -})(); -""" - - -class BrowserRecorder(RecorderBase): - capability_map = { - "click": "implemented", - "fill": "implemented", - "press_key": "implemented", - "navigation": "partial", - } - - def __init__( - self, - *, - on_event: Optional[Callable[[RawUserEvent], None]] = None, - on_raw_event: Optional[Callable[[dict[str, Any]], None]] = None, - poll_sec: float = 0.25, - ) -> None: - super().__init__(on_event=on_event) - self.on_raw_event = on_raw_event - self.poll_sec = poll_sec - self._last_url = "" - self._last_urls_by_page: dict[str, str] = {} - self._installed_pages: set[int] = set() - self._bound_events: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=512) - self._seen_event_ids: set[str] = set() - self._seen_event_order: list[str] = [] - self._known_page_ids: set[str] = set() - self._pending_navigations: dict[str, dict[str, Any]] = {} - self._pending_pointerdowns: dict[str, dict[str, Any]] = {} - self._navigation_reorder_sec = 0.45 - self._pointer_fallback_sec = 0.35 - - @property - def capability(self) -> dict[str, str]: - return dict(self.capability_map) - - def start(self) -> None: - if self._thread and self._thread.is_alive(): - self.status = RecorderState.RECORDING - return - self._stop.clear() - self.status = RecorderState.RECORDING - try: - import kit_browser - - kit_browser.install_event_bridge(_LISTEN_SCRIPT, self._receive_bound_event) - initial_ids = kit_browser.call_on_each_open_page( - lambda page: kit_browser.page_id_for(page) - ) - self._known_page_ids.update( - str(page_id) for page_id in initial_ids if page_id - ) - except Exception: - # Polling remains as a compatibility fallback. - pass - self._thread = threading.Thread( - target=self._loop, name="ui-inspector-browser-recorder", daemon=True - ) - self._thread.start() - - def stop(self) -> None: - self._stop.set() - if self._thread and self._thread.is_alive(): - self._thread.join(timeout=2.0) - self._thread = None - # Disable the bridge before draining its local queue so an event that - # arrived just before Stop is not silently lost, while no new events - # can race with the drain. - try: - import kit_browser - - kit_browser.clear_event_bridge() - except Exception: - pass - if self.status == RecorderState.RECORDING: - try: - self._drain_bound_events() - self._flush_pending_pointerdowns(force=True) - self._flush_pending_navigations(force=True) - except Exception: - pass - self.status = RecorderState.STOPPED - - def _receive_bound_event(self, event: dict[str, Any]) -> None: - try: - self._bound_events.put_nowait(dict(event or {})) - except queue.Full: - try: - self._bound_events.get_nowait() - self._bound_events.put_nowait(dict(event or {})) - except (queue.Empty, queue.Full): - pass - - def _loop(self) -> None: - while not self._stop.is_set(): - if self.status == RecorderState.PAUSED: - time.sleep(0.1) - continue - if self.status != RecorderState.RECORDING: - time.sleep(0.05) - continue - try: - self._tick() - except Exception: - pass - time.sleep(self.poll_sec) - - def _tick(self) -> None: - import kit_browser - - self._drain_bound_events() - - # Install + drain on every open tab (detail may be a second tab). - payloads = kit_browser.call_on_each_open_page(self._install_and_drain) - if payloads: - for payload in payloads: - if not isinstance(payload, dict): - continue - page_id = str(payload.get("page_id") or "") - url = str(payload.get("url") or "") - previous_url = self._last_urls_by_page.get(page_id, "") if page_id else "" - if previous_url and url and url != previous_url: - self._schedule_navigation( - page_id=page_id, from_url=previous_url, to_url=url - ) - elif ( - page_id - and page_id not in self._known_page_ids - and self._known_page_ids - and url - and not url.startswith("about:") - ): - opener_id = str(payload.get("opener_page_id") or "") - self._schedule_navigation( - page_id=page_id, - from_url=self._last_urls_by_page.get(opener_id, ""), - to_url=url, - opener_page_id=opener_id, - new_page=True, - ) - if page_id and url: - self._last_urls_by_page[page_id] = url - self._known_page_ids.add(page_id) - for ev in payload.get("events") or []: - if isinstance(ev, dict) and not ev.get("page_id"): - ev["page_id"] = payload.get("page_id") or "" - self._emit_dom_event(ev) - - self._flush_pending_pointerdowns() - - # Navigation is intentionally emitted after both bridge and polling - # queues have been drained. This preserves click -> navigation order - # even when the context bridge is unavailable and polling is used. - self._flush_pending_navigations() - - def _drain_bound_events(self) -> None: - while True: - try: - self._emit_dom_event(self._bound_events.get_nowait()) - except queue.Empty: - break - - def _schedule_navigation( - self, - *, - page_id: str, - from_url: str, - to_url: str, - opener_page_id: str = "", - new_page: bool = False, - ) -> None: - self._pending_navigations[page_id] = { - "due": time.monotonic() + self._navigation_reorder_sec, - "from": from_url, - "to": to_url, - "page_id": page_id, - "opener_page_id": opener_page_id, - "new_page": new_page, - } - - def _flush_pending_navigations(self, *, force: bool = False) -> None: - now = time.monotonic() - ready = [ - page_id - for page_id, item in self._pending_navigations.items() - if force or float(item.get("due") or 0.0) <= now - ] - for page_id in ready: - self._flush_pending_pointerdowns(force=True, page_id=page_id) - item = self._pending_navigations.pop(page_id, None) or {} - self.emit( - RawUserEvent( - kind="navigation", - surface="browser", - url=str(item.get("to") or ""), - meta={ - "from": item.get("from") or "", - "to": item.get("to") or "", - "page_id": page_id, - "opener_page_id": item.get("opener_page_id") or "", - "new_page": bool(item.get("new_page")), - }, - ) - ) - - def _install_and_drain(self, page: Any) -> dict[str, Any]: - """Runs on Playwright worker thread (once per open tab).""" - try: - page.evaluate(_LISTEN_SCRIPT) - except Exception: - pass - # Also try frames - try: - for frame in page.frames: - try: - frame.evaluate(_LISTEN_SCRIPT) - except Exception: - pass - except Exception: - pass - events = [] - try: - events = page.evaluate( - """() => { - const e = window.__openclawUiInspectorEvents || []; - window.__openclawUiInspectorEvents = []; - return e; - }""" - ) or [] - except Exception: - events = [] - url = "" - try: - url = page.url or "" - except Exception: - url = "" - page_id = "" - opener_page_id = "" - try: - import kit_browser - - page_id = kit_browser.page_id_for(page) - try: - opener = page.opener() - if opener is not None: - opener_page_id = kit_browser.page_id_for(opener) - except Exception: - pass - except Exception: - pass - return { - "url": url, - "page_id": page_id, - "opener_page_id": opener_page_id, - "events": events, - } - - def _emit_dom_event(self, ev: dict[str, Any]) -> None: - event_id = str(ev.get("event_id") or "") - if event_id: - if event_id in self._seen_event_ids: - return - self._seen_event_ids.add(event_id) - self._seen_event_order.append(event_id) - if len(self._seen_event_order) > 2000: - old = self._seen_event_order.pop(0) - self._seen_event_ids.discard(old) - if self.on_raw_event: - try: - self.on_raw_event(dict(ev)) - except Exception: - pass - kind = str(ev.get("kind") or "") - hit_target = dict(ev.get("hit_target") or ev.get("target") or {}) - candidate_chain = list(ev.get("candidate_chain") or ev.get("ancestry") or []) - if not candidate_chain: - candidate_chain = [hit_target] - action_hint = dict(ev.get("action_target") or {}) - if action_hint and action_hint != hit_target: - candidate_chain.append(action_hint) - try: - from core.policies.target_scoring import resolve_browser_target - - target, target_scores = resolve_browser_target( - hit_target, - candidate_chain, - x=int(ev.get("x")) if ev.get("x") is not None else None, - y=int(ev.get("y")) if ev.get("y") is not None else None, - ) - except Exception: - target = dict(ev.get("action_target") or ev.get("target") or hit_target) - target_scores = [] - if candidate_chain: - target["parent_chain"] = candidate_chain - frame = { - "url": ev.get("frame_url") or "", - "name": ev.get("frame_name") or "", - "id": ev.get("frame_id") or "", - } - if kind in {"pointerdown", "click"}: - raw = RawUserEvent( - kind="click", - surface="browser", - target=target, - frame=frame, - url=str(ev.get("url") or ""), - x=int(ev.get("x")) if ev.get("x") is not None else None, - y=int(ev.get("y")) if ev.get("y") is not None else None, - meta={ - "button": ev.get("button"), - "event_id": event_id, - "document_id": ev.get("document_id"), - "page_id": ev.get("page_id"), - "frame_id": ev.get("frame_id"), - "sequence": ev.get("sequence"), - "source_ts_ms": ev.get("ts"), - "hit_target": ev.get("hit_target") or {}, - "action_target": ev.get("action_target") or target, - "target_scores": target_scores, - "captured_from": kind, - }, - ) - if kind == "pointerdown": - key = self._pointer_key(ev) - self._pending_pointerdowns[key] = { - "due": time.monotonic() + self._pointer_fallback_sec, - "event": raw, - "page_id": str(ev.get("page_id") or ""), - } - return - self._cancel_pointerdown_for_click(ev) - self.emit(raw) - elif kind == "input": - self.emit( - RawUserEvent( - kind="input", - surface="browser", - text=str(ev.get("text") or target.get("value") or ""), - target=target, - frame=frame, - url=str(ev.get("url") or ""), - meta={ - "input_type": target.get("input_type") or "", - "event_id": event_id, - "document_id": ev.get("document_id"), - "page_id": ev.get("page_id"), - "frame_id": ev.get("frame_id"), - "sequence": ev.get("sequence"), - "source_ts_ms": ev.get("ts"), - "input_event": ev.get("input_event") or "input", - "is_trusted": bool(ev.get("is_trusted")), - }, - ) - ) - elif kind == "key": - self.emit( - RawUserEvent( - kind="key", - surface="browser", - key=str(ev.get("key") or ""), - target=target, - frame=frame, - url=str(ev.get("url") or ""), - meta={ - "event_id": event_id, - "document_id": ev.get("document_id"), - "page_id": ev.get("page_id"), - "frame_id": ev.get("frame_id"), - "sequence": ev.get("sequence"), - "source_ts_ms": ev.get("ts"), - }, - ) - ) - - @staticmethod - def _pointer_key(ev: dict[str, Any]) -> str: - return "|".join( - str(ev.get(key) or "") - for key in ("page_id", "document_id", "pointer_id", "button") - ) - - def _cancel_pointerdown_for_click(self, ev: dict[str, Any]) -> None: - exact = self._pointer_key(ev) - if exact in self._pending_pointerdowns: - self._pending_pointerdowns.pop(exact, None) - return - page_id = str(ev.get("page_id") or "") - button = str(ev.get("button") or "") - for key, item in list(self._pending_pointerdowns.items()): - event = item.get("event") - meta = getattr(event, "meta", {}) or {} - if str(meta.get("page_id") or "") == page_id and str(meta.get("button") or "") == button: - self._pending_pointerdowns.pop(key, None) - return - - def _flush_pending_pointerdowns( - self, *, force: bool = False, page_id: str = "" - ) -> None: - now = time.monotonic() - ready = [ - key for key, item in self._pending_pointerdowns.items() - if (not page_id or item.get("page_id") == page_id) - and (force or float(item.get("due") or 0.0) <= now) - ] - for key in ready: - item = self._pending_pointerdowns.pop(key, None) or {} - event = item.get("event") - if event is not None: - event.meta["pointerdown_fallback"] = True - self.emit(event) diff --git a/tools/ui_inspector/recorder/input_aggregator.py b/tools/ui_inspector/recorder/input_aggregator.py deleted file mode 100644 index adc9bee..0000000 --- a/tools/ui_inspector/recorder/input_aggregator.py +++ /dev/null @@ -1,234 +0,0 @@ -"""Aggregate keystrokes / value changes into fill/type + press_key.""" - -from __future__ import annotations - -import time -from dataclasses import dataclass, field -from typing import Any, Optional - -from redaction import redact_input_payload - -# Keys recorded as independent press_key actions. -_FUNCTION_KEYS = { - "Enter", - "Escape", - "Tab", - "Backspace", - "Delete", - "ArrowUp", - "ArrowDown", - "ArrowLeft", - "ArrowRight", - "Home", - "End", - "PageUp", - "PageDown", - "F1", - "F2", - "F3", - "F4", - "F5", - "F6", - "F7", - "F8", - "F9", - "F10", - "F11", - "F12", -} - - -@dataclass -class AggregatedInput: - action_type: str # fill | type | press_key - text: str = "" - key: str = "" - target: dict[str, Any] = field(default_factory=dict) - sensitive: bool = False - input_payload: dict[str, Any] = field(default_factory=dict) - started_at: float = 0.0 - ended_at: float = 0.0 - capture_context: dict[str, Any] = field(default_factory=dict) - raw_event: dict[str, Any] = field(default_factory=dict) - raw_event_ids: list[str] = field(default_factory=list) - - -class InputAggregator: - """ - Do not emit one Action per key. - Observe focus element value changes; flush after idle gap or function key. - """ - - def __init__(self, *, idle_sec: float = 0.8) -> None: - self.idle_sec = idle_sec - self._buf_text = "" - self._target: dict[str, Any] = {} - self._started: float = 0.0 - self._last: float = 0.0 - self._input_type = "" - self._field_name = "" - self._capture_context: dict[str, Any] = {} - self._raw_event: dict[str, Any] = {} - self._raw_event_ids: list[str] = [] - self._target_identity = "" - - @staticmethod - def _stable_target_identity( - target: dict[str, Any], capture_context: dict[str, Any] - ) -> str: - from core.policies.element_identity import element_identity_key - - return element_identity_key( - target or {}, - surface=str(capture_context.get("surface") or "windows"), - page_url=str(capture_context.get("url") or ""), - frame_name=str(capture_context.get("frame_name") or ""), - ) - - @staticmethod - def _event_id(raw_event: Optional[dict[str, Any]]) -> str: - return str(((raw_event or {}).get("meta") or {}).get("event_id") or "") - - def on_key( - self, - *, - key: str, - text: str = "", - target: Optional[dict[str, Any]] = None, - input_type: str = "", - field_name: str = "", - capture_context: Optional[dict[str, Any]] = None, - raw_event: Optional[dict[str, Any]] = None, - value_event: bool = False, - now: Optional[float] = None, - ) -> list[AggregatedInput]: - now = time.monotonic() if now is None else now - out: list[AggregatedInput] = [] - k = key or "" - if k in _FUNCTION_KEYS or (len(k) > 1 and k.startswith("F") and k[1:].isdigit()): - out.extend(self.flush(now=now)) - payload, sens = redact_input_payload( - {"key": k}, - field_name=field_name or self._field_name, - input_type=input_type or self._input_type, - ) - out.append( - AggregatedInput( - action_type="press_key", - key=k, - target=dict(target or self._target or {}), - sensitive=sens, - input_payload=payload, - started_at=now, - ended_at=now, - capture_context=dict(capture_context or self._capture_context or {}), - raw_event=dict(raw_event or {}), - raw_event_ids=( - [self._event_id(raw_event)] if self._event_id(raw_event) else [] - ), - ) - ) - return out - - # Printable / value change - if target: - # Focus changed → flush previous - next_context = dict(capture_context or self._capture_context or {}) - next_identity = self._stable_target_identity(target, next_context) - if self._target_identity and next_identity != self._target_identity: - out.extend(self.flush(now=now)) - self._target = dict(target) - self._target_identity = next_identity - if input_type: - self._input_type = input_type - if field_name: - self._field_name = field_name - if capture_context: - self._capture_context = dict(capture_context) - if raw_event: - self._raw_event = dict(raw_event) - event_id = self._event_id(raw_event) - if event_id and event_id not in self._raw_event_ids: - self._raw_event_ids.append(event_id) - - chunk = text if text else (k if len(k) == 1 else "") - if not chunk and not text and not value_event: - return out - - if not self._started: - self._started = now - # If observer sends full value, replace buffer; else append char - if value_event: - self._buf_text = text - elif text and len(text) >= len(self._buf_text): - self._buf_text = text - else: - self._buf_text += chunk - self._last = now - return out - - def on_value_change( - self, - *, - value: str, - target: Optional[dict[str, Any]] = None, - input_type: str = "", - field_name: str = "", - capture_context: Optional[dict[str, Any]] = None, - raw_event: Optional[dict[str, Any]] = None, - now: Optional[float] = None, - ) -> list[AggregatedInput]: - return self.on_key( - key="", - text=value, - target=target, - input_type=input_type, - field_name=field_name, - capture_context=capture_context, - raw_event=raw_event, - value_event=True, - now=now, - ) - - def flush(self, *, now: Optional[float] = None) -> list[AggregatedInput]: - now = ( - (time.time() if self._last > 100_000_000 else time.monotonic()) - if now is None else now - ) - if not self._buf_text and not self._started: - return [] - payload, sens = redact_input_payload( - {"text": self._buf_text, "name": self._field_name}, - field_name=self._field_name, - input_type=self._input_type, - ) - item = AggregatedInput( - action_type="fill", - text=str(payload.get("text") or ""), - target=dict(self._target or {}), - sensitive=sens, - input_payload=payload, - started_at=self._started or now, - ended_at=now, - capture_context=dict(self._capture_context or {}), - raw_event=dict(getattr(self, "_raw_event", {}) or {}), - raw_event_ids=list(self._raw_event_ids), - ) - self._buf_text = "" - self._started = 0.0 - self._last = 0.0 - self._input_type = "" - self._capture_context = {} - self._raw_event = {} - self._raw_event_ids = [] - # keep target for next session - return [item] - - def maybe_flush_idle(self, *, now: Optional[float] = None) -> list[AggregatedInput]: - now = ( - (time.time() if self._last > 100_000_000 else time.monotonic()) - if now is None else now - ) - if self._last and (now - self._last) >= self.idle_sec and self._started: - return self.flush(now=now) - return [] diff --git a/tools/ui_inspector/recorder/manager.py b/tools/ui_inspector/recorder/manager.py deleted file mode 100644 index c7d4c96..0000000 --- a/tools/ui_inspector/recorder/manager.py +++ /dev/null @@ -1,1220 +0,0 @@ -"""Action recording manager: before → action → after, dual-write v1/v2.""" - -from __future__ import annotations - -import threading -import time -from datetime import datetime -from pathlib import Path -import html -import re -from typing import Any, Callable, Optional - -from locators.catalog import ElementCatalogService -from models.action import ACTION_CAPABILITY, ActionFlow, ActionRecord -from models.serialize import to_dict, write_model_json -from redaction import redact_input_payload, sanitize_artifact -from snapshot_service import SnapshotService -from state_diff import ( - changes_to_dicts, - diff_states, - postconditions_to_dicts, - propose_postconditions, -) -from core.policies.element_identity import element_identity_key - -from .base import RawUserEvent, RecorderState -from .input_aggregator import InputAggregator - - -def _now_iso() -> str: - return datetime.now().astimezone().isoformat() - - -class ActionRecordingManager: - """ - Central coordinator. - - - Action is the flow spine - - States are before/after evidence (fingerprint-deduped) - - Dual-writes optional v1 step via callback - """ - - def __init__( - self, - flow_dir: Path, - *, - flow_name: str = "", - flow_id: str = "", - surface: str = "windows", - recording_mode: str = "auto", - dump_fn: Optional[Callable[[], Any]] = None, - write_v1_step: Optional[Callable[..., Any]] = None, - stability_timeout_sec: float = 3.0, - save_sensitive_screenshots: bool = False, - defer_action_evidence: bool = False, - on_action: Optional[Callable[[ActionRecord], None]] = None, - on_action_update: Optional[Callable[[ActionRecord, str], None]] = None, - ) -> None: - self.flow_dir = Path(flow_dir) - self.actions_dir = self.flow_dir / "actions" - self.actions_dir.mkdir(parents=True, exist_ok=True) - self.snapshots = SnapshotService( - self.flow_dir, - save_sensitive_screenshots=save_sensitive_screenshots, - ) - self.catalog = ElementCatalogService(self.flow_dir, flow_id=flow_id or flow_name) - self.aggregator = InputAggregator(idle_sec=8.0 if surface == "browser" else 0.8) - self.dump_fn = dump_fn - self.defer_action_evidence = bool(defer_action_evidence) - self.on_action = on_action - self.on_action_update = on_action_update - self.write_v1_step = write_v1_step - self.stability_timeout_sec = stability_timeout_sec - self._lock = threading.RLock() - self._seq = 0 - self.status = RecorderState.IDLE - self.last_action: Optional[ActionRecord] = None - self.last_warning: str = "" - self.action_flow = ActionFlow( - flow_id=flow_id or self.flow_dir.name, - flow_name=flow_name or self.flow_dir.name, - recording_mode=recording_mode, - surface=surface, - metadata={"action_capability": dict(ACTION_CAPABILITY)}, - ) - self._debounce_until = 0.0 - self._debounce_sec = 0.35 - self._last_click_signature = "" - self._last_click_monotonic = 0.0 - self._actions_by_id: dict[str, ActionRecord] = {} - self._last_fill: dict[str, tuple[str, float]] = {} - self._last_text_target: dict[str, Any] = {} - - def _dump_for_context(self, context: Optional[dict[str, Any]] = None) -> Any: - if not self.dump_fn: - return None - ctx = dict(context or {}) - kwargs = { - key: ctx.get(key) - for key in ("page_id", "document_id", "frame_id") - if ctx.get(key) - } - if kwargs: - try: - return self.dump_fn(**kwargs) - except TypeError as exc: - if "unexpected keyword" not in str(exc) and "positional" not in str(exc): - raise - return self.dump_fn() - - @staticmethod - def _event_context(event: RawUserEvent) -> dict[str, Any]: - meta = dict(event.meta or {}) - return { - "surface": event.surface or "", - "page_id": meta.get("page_id") or "", - "document_id": meta.get("document_id") or "", - "frame_id": meta.get("frame_id") or "", - "url": event.url or "", - "frame_url": (event.frame or {}).get("url") or "", - "frame_name": (event.frame or {}).get("name") or "", - "source_ts_ms": meta.get("source_ts_ms"), - "event_id": meta.get("event_id") or "", - "sequence": meta.get("sequence"), - "timestamp": event.timestamp, - "input_event": meta.get("input_event") or "", - "is_trusted": meta.get("is_trusted"), - } - - @staticmethod - def _captured_live_counts(target: dict[str, Any]) -> Optional[dict[str, int]]: - """Use uniqueness measured synchronously in the event-origin document.""" - counts: dict[str, int] = {} - for strategy, field in ( - ("css", "css_count"), - ("stable_css", "stable_css_count"), - ("href", "href_count"), - ): - value = (target or {}).get(field) - if value is not None: - counts[strategy] = int(value) - return counts or None - - def _stable_capture( - self, context: dict[str, Any], *, reason: str, timeout: float - ): - return self.snapshots.wait_until_stable( - lambda: self._dump_for_context(context), - timeout_sec=timeout, - change_reason=reason, - ) - - def _action_capture(self, context: dict[str, Any], *, reason: str): - """Capture one post-action observation without blocking on two dumps. - - Browser user events can arrive faster than a full page dump. A second - stability poll made the FIFO lag by several seconds and contaminated - earlier actions with later input. The action stream keeps ordering; - quiet/checkpoint capture remains available for deliberately stable UI. - """ - import time - - time.sleep(0.06) - dump = self._dump_for_context(context) - return self.snapshots.capture_from_dump( - dump, change_reason=reason, force_persist=True - ) - - @staticmethod - def _enrich_weak_browser_target(target: dict[str, Any], before) -> dict[str, Any]: - """Attach a semantic hint without discarding the physical hit target.""" - out = dict(target or {}) - role = str(out.get("role") or "").lower() - if role not in {"", "path", "svg", "img", "span", "div"}: - return out - if str(out.get("name") or out.get("text") or "").strip(): - return out - scope_text = " ".join( - [str(out.get("css") or "")] - + [ - " ".join( - str((ancestor or {}).get(key) or "") - for key in ("id", "class_name", "css") - ) - for ancestor in (out.get("parent_chain") or []) - ] - ).lower() - form_hints = ( - "comment", "editor", "input", "form", "reply", "message", "chat", - "评论", "输入", "回复", "消息", - ) - if not any(token in scope_text for token in form_hints): - return out - action_names = { - "submit", "send", "publish", "post", "reply", "comment", - "提交", "发送", "发布", "回复", "评论", - "送信", "投稿", "公開", "보내기", "게시", "enviar", "publicar", - "envoyer", "publier", "senden", "veröffentlichen", - } - candidates = [] - for row in list(getattr(before, "locatable", None) or []): - row_role = str(row.get("role") or row.get("control_type") or "").lower() - row_name = str(row.get("name") or row.get("value") or row.get("text") or "").strip() - if row_role in {"button", "menuitem", "link"} and row_name.lower() in action_names: - candidates.append(row) - unique = { - (str(row.get("role") or row.get("control_type") or ""), str(row.get("name") or row.get("value") or row.get("text") or "")) - for row in candidates - } - if len(unique) == 1: - semantic_role, semantic_name = next(iter(unique)) - out["semantic_role"] = semantic_role - out["semantic_name"] = semantic_name - out["semantic_inference"] = "form_scope_unique_action" - out["semantic_confidence"] = 0.72 - return out - - def start(self, *, entry_dump: Optional[Any] = None) -> ActionRecord | None: - with self._lock: - self.status = RecorderState.RECORDING - self.action_flow.status = "open" - self.action_flow.created_at = _now_iso() - dump = entry_dump - if dump is None and self.dump_fn: - try: - dump = self._dump_for_context() - except Exception as exc: - self.last_warning = f"entry dump failed: {exc}" - dump = None - if dump is not None: - st = self.snapshots.capture_from_dump( - dump, change_reason="entry", force_persist=True - ) - self.action_flow.entry_state_id = st.id - self.action_flow.final_state_id = st.id - if st.id not in self.action_flow.states: - self.action_flow.states.append(st.id) - if self.snapshots.latest_observation: - self.action_flow.observations.append( - self.snapshots.latest_observation.id - ) - # Optional v1 step for entry checkpoint - if self.write_v1_step: - try: - self.write_v1_step(dump, label_hint="entry", change_reason="entry") - except Exception: - pass - self._persist_flow() - return None - - def pause(self) -> None: - self.status = RecorderState.PAUSED - - def resume(self) -> None: - if self.status == RecorderState.PAUSED: - self.status = RecorderState.RECORDING - - def stop(self) -> ActionFlow: - with self._lock: - for item in self.aggregator.flush(): - self._commit_fill(item) - self.status = RecorderState.STOPPED - self.action_flow.status = "ended" - self.action_flow.ended_at = _now_iso() - if self.snapshots.latest: - self.action_flow.final_state_id = self.snapshots.latest.id - self._persist_flow() - self.catalog.save() - return self.action_flow - - def handle_raw_event(self, event: RawUserEvent) -> Optional[ActionRecord]: - if self.status != RecorderState.RECORDING: - return None - if event.from_inspector: - return None - with self._lock: - kind = event.kind - if kind in {"mouse_down", "click"}: - return self._handle_click(event) - if kind in {"key", "input", "value_change"}: - return self._handle_input(event) - if kind == "navigation": - return self._handle_navigation(event) - if kind == "checkpoint": - return self.record_checkpoint(meta=event.meta) - return None - - def record_checkpoint(self, *, meta: Optional[dict[str, Any]] = None) -> Optional[ActionRecord]: - with self._lock: - before = self.snapshots.latest - before_obs = self.snapshots.latest_observation - dump = None - if self.dump_fn: - try: - dump = self._dump_for_context(meta) - except Exception as exc: - return self._failed_action( - "checkpoint", error=str(exc), before=before - ) - after = self.snapshots.capture_from_dump( - dump, change_reason="checkpoint", force_persist=True - ) - return self._finalize_action( - action_type="checkpoint", - before=before, - after=after, - target={}, - input_payload={}, - raw_event={"meta": meta or {}}, - confidence=0.9, - before_observation_id=before_obs.id if before_obs else None, - after_observation_id=( - self.snapshots.latest_observation.id - if self.snapshots.latest_observation else None - ), - ) - - def _handle_click(self, event: RawUserEvent) -> Optional[ActionRecord]: - import time - - now = time.monotonic() - context = self._event_context(event) - target = dict(event.target or {}) - click_identity = element_identity_key( - target, - surface=event.surface, - page_url=str(context.get("url") or ""), - frame_name=str(context.get("frame_name") or ""), - ) - signature = "|".join( - (click_identity, str(event.button or ""), str(event.x), str(event.y)) - ) - if ( - signature == self._last_click_signature - and now - self._last_click_monotonic < self._debounce_sec - ): - return None - self._last_click_signature = signature - self._last_click_monotonic = now - # Flush pending input before click - for item in self.aggregator.flush(): - self._commit_fill(item) - - before = self.snapshots.latest - before_obs = self.snapshots.latest_observation - if event.surface == "browser": - target = self._enrich_weak_browser_target(target, before) - element = None - live_counts = self._captured_live_counts(target) - if ( - not self.defer_action_evidence - and event.surface == "browser" and context.get("page_id") and target - ): - try: - import kit_browser - - live_counts = kit_browser.validate_element_on_page( - page_id=str(context.get("page_id") or ""), - element=target, - frame=event.frame or None, - ) - except Exception as exc: - self.last_warning = f"live locator validation failed: {exc}" - if target: - try: - element = self.catalog.upsert_from_snapshot( - target, - surface=event.surface, - window_scope=target.get("window_scope") or {}, - rows=(before.locatable if before else None), - frame=event.frame or None, - live_counts=live_counts, - ) - except Exception as exc: - self.last_warning = f"catalog upsert failed: {exc}" - - after = before - status = "ok" - err = "" - if self.dump_fn and not self.defer_action_evidence: - try: - after = self._action_capture(context, reason="after_click") - except Exception as exc: - status = "partial" - err = str(exc) - try: - dump = self._dump_for_context(context) - after = self.snapshots.capture_from_dump( - dump, change_reason="after_click_partial", force_persist=True - ) - except Exception as exc2: - status = "failed" - err = f"{err}; {exc2}" - - act = self._finalize_action( - action_type="click", - before=before, - after=after, - target=target, - target_element_id=element.id if element else None, - input_payload={}, - raw_event=to_dict(event) if hasattr(event, "__dataclass_fields__") else dict(event.__dict__), - status=status, - confidence=0.8 if status == "ok" else 0.4, - evidence={"error": err} if err else {}, - started_at=event.timestamp, - before_observation_id=before_obs.id if before_obs else None, - after_observation_id=( - self.snapshots.latest_observation.id - if self.snapshots.latest_observation else None - ), - ) - if self.write_v1_step and after is not None: - # Dual-write one v1 step per meaningful action (not heartbeat) - try: - # Reconstruct a minimal dump-like for v1 from after state paths is hard; - # caller should pass dump through write_v1_step if needed. - pass - except Exception: - pass - return act - - def _handle_input(self, event: RawUserEvent) -> Optional[ActionRecord]: - key = event.key or (event.meta or {}).get("key") or "" - text = event.text or (event.meta or {}).get("text") or (event.meta or {}).get("value") or "" - common = { - "target": event.target, - "input_type": str((event.meta or {}).get("input_type") or ""), - "field_name": str((event.target or {}).get("name") or ""), - "capture_context": self._event_context(event), - "raw_event": to_dict(event), - } - source_ms = (event.meta or {}).get("source_ts_ms") - event_clock = ( - float(source_ms) / 1000.0 if source_ms is not None else time.monotonic() - ) - if event.kind in {"input", "value_change"}: - items = self.aggregator.on_value_change(value=text, now=event_clock, **common) - else: - items = self.aggregator.on_key(key=key, text=text, now=event_clock, **common) - items.extend(self.aggregator.maybe_flush_idle(now=event_clock)) - last = None - for item in items: - if item.action_type == "press_key": - last = self._commit_press_key(item) - else: - last = self._commit_fill(item) - return last - - def _handle_navigation(self, event: RawUserEvent) -> Optional[ActionRecord]: - before = self.snapshots.latest - before_obs = self.snapshots.latest_observation - context = self._event_context(event) - after = before - if self.dump_fn and not self.defer_action_evidence: - try: - after = self._action_capture(context, reason="navigation") - except Exception: - pass - return self._finalize_action( - action_type="navigation", - before=before, - after=after, - target={}, - input_payload={"url": event.url}, - raw_event={"url": event.url, "meta": event.meta}, - confidence=0.7, - started_at=event.timestamp, - before_observation_id=before_obs.id if before_obs else None, - after_observation_id=(self.snapshots.latest_observation.id if self.snapshots.latest_observation else None), - ) - - def _commit_fill(self, item) -> Optional[ActionRecord]: - if ( - item.target - and (self.action_flow.surface or "") == "windows" - and not self._is_oversized_windows_target(item.target) - ): - self._last_text_target = dict(item.target) - page_url = str(item.capture_context.get("url") or "") - identity = element_identity_key( - item.target or {}, - surface=self.action_flow.surface or "windows", - page_url=page_url, - frame_name=str(item.capture_context.get("frame_name") or ""), - ) - text_value = str((item.input_payload or {}).get("text") or "") - previous = self._last_fill.get(identity) - source_ms = item.capture_context.get("source_ts_ms") - now_marker = float(source_ms) / 1000.0 if source_ms else time.time() - if previous and previous[0] == text_value and now_marker - previous[1] <= 15.0: - return None - self._last_fill[identity] = (text_value, now_marker) - before = self.snapshots.latest - before_obs = self.snapshots.latest_observation - after = before - if self.dump_fn and not self.defer_action_evidence: - try: - after = self._action_capture( - item.capture_context, reason="after_fill" - ) - except Exception: - pass - element = None - frame = { - "url": item.capture_context.get("frame_url") or "", - "name": item.capture_context.get("frame_name") or "", - } - live_counts = self._captured_live_counts(item.target or {}) - if ( - not self.defer_action_evidence - and item.capture_context.get("page_id") and item.target - ): - try: - import kit_browser - - live_counts = kit_browser.validate_element_on_page( - page_id=str(item.capture_context.get("page_id") or ""), - element=item.target, - frame=frame, - ) - except Exception as exc: - self.last_warning = f"live fill locator validation failed: {exc}" - if item.target: - try: - element = self.catalog.upsert_from_snapshot( - item.target, - surface=self.action_flow.surface or "windows", - window_scope=item.target.get("window_scope") or {}, - rows=(before.locatable if before else None), - frame=frame, - live_counts=live_counts, - ) - except Exception: - pass - return self._finalize_action( - action_type="fill", - before=before, - after=after, - target=item.target, - target_element_id=element.id if element else None, - input_payload=item.input_payload, - sensitive=item.sensitive, - confidence=0.75, - raw_event=item.raw_event or { - "url": item.capture_context.get("url") or "", - "meta": dict(item.capture_context or {}), - }, - raw_event_ids=item.raw_event_ids, - started_at=str(item.capture_context.get("timestamp") or _now_iso()), - before_observation_id=before_obs.id if before_obs else None, - after_observation_id=(self.snapshots.latest_observation.id if self.snapshots.latest_observation else None), - ) - - def _commit_press_key(self, item) -> ActionRecord: - target = dict(item.target or {}) - if ( - (self.action_flow.surface or "") == "windows" - and self._is_oversized_windows_target(target) - and self._last_text_target - and self._same_windows_target_scope(target, self._last_text_target) - ): - target = dict(self._last_text_target) - before = self.snapshots.latest - before_obs = self.snapshots.latest_observation - after = before - if self.dump_fn and not self.defer_action_evidence: - try: - after = self._action_capture( - item.capture_context, reason="after_key" - ) - except Exception: - pass - element = None - frame = { - "url": item.capture_context.get("frame_url") or "", - "name": item.capture_context.get("frame_name") or "", - } - live_counts = self._captured_live_counts(target) - if target: - try: - element = self.catalog.upsert_from_snapshot( - target, - surface=self.action_flow.surface or "windows", - window_scope=target.get("window_scope") or {}, - rows=(before.locatable if before else None), - frame=frame, - live_counts=live_counts, - ) - except Exception as exc: - self.last_warning = f"key target catalog upsert failed: {exc}" - return self._finalize_action( - action_type="press_key", - before=before, - after=after, - target=target, - target_element_id=element.id if element else None, - input_payload=item.input_payload, - sensitive=item.sensitive, - confidence=0.7, - raw_event=item.raw_event or { - "kind": "key", - "surface": self.action_flow.surface or "", - "key": item.key, - "target": target, - "frame": frame, - "url": item.capture_context.get("url") or "", - "meta": dict(item.capture_context or {}), - }, - raw_event_ids=item.raw_event_ids, - started_at=str(item.capture_context.get("timestamp") or _now_iso()), - before_observation_id=before_obs.id if before_obs else None, - after_observation_id=(self.snapshots.latest_observation.id if self.snapshots.latest_observation else None), - ) - - def _failed_action( - self, action_type: str, *, error: str, before - ) -> ActionRecord: - return self._finalize_action( - action_type=action_type, - before=before, - after=before, - target={}, - input_payload={}, - status="failed", - confidence=0.1, - evidence={"error": error}, - ) - - def _finalize_action( - self, - *, - action_type: str, - before, - after, - target: dict[str, Any], - input_payload: dict[str, Any], - target_element_id: Optional[str] = None, - sensitive: bool = False, - raw_event: Optional[dict[str, Any]] = None, - raw_event_ids: Optional[list[str]] = None, - status: str = "ok", - confidence: float = 0.5, - evidence: Optional[dict[str, Any]] = None, - before_observation_id: Optional[str] = None, - after_observation_id: Optional[str] = None, - started_at: Optional[str] = None, - ) -> ActionRecord: - self._seq += 1 - payload, sens2 = redact_input_payload(input_payload or {}) - sensitive = sensitive or sens2 - clean_target = sanitize_artifact(target or {}, sensitive_hint=sensitive) - clean_raw_event = sanitize_artifact(raw_event or {}, sensitive_hint=sensitive) - - before_id = before.id if before else None - after_id = after.id if after else before_id - if self.defer_action_evidence and action_type != "checkpoint": - # A slow eventual dump must not be presented as an event-time - # after-state. The evidence coordinator attaches it explicitly. - after_id = None - after_observation_id = None - if after and after.id not in self.action_flow.states: - self.action_flow.states.append(after.id) - if before and before.id not in self.action_flow.states: - self.action_flow.states.append(before.id) - - observed = [] - proposed = [] - if before and after: - b = self.snapshots.to_summary_dict(before) - a = self.snapshots.to_summary_dict(after) - changes = diff_states(b, a) - observed = changes_to_dicts(changes) - proposed = postconditions_to_dicts(propose_postconditions(changes)) - - # Attach locator uniqueness hint for UI - match_count = None - unique = None - primary_expr = "" - if target_element_id: - for el in self.catalog.catalog.elements: - if el.id == target_element_id and el.primary_locator: - match_count = el.primary_locator.actual_count - unique = el.primary_locator.unique - primary_expr = el.primary_locator.expression - break - if target_element_id not in self.action_flow.elements: - self.action_flow.elements.append(target_element_id) - - if raw_event_ids is None: - event_id = str(((clean_raw_event or {}).get("meta") or {}).get("event_id") or "") - raw_event_ids = [event_id] if event_id else [] - act = ActionRecord( - sequence=self._seq, - action_type=action_type, - started_at=started_at or _now_iso(), - ended_at=_now_iso(), - surface=self.action_flow.surface or "", - target_element_id=target_element_id, - target_snapshot=dict(clean_target or {}), - input=payload, - sensitive=sensitive, - before_state_id=before_id, - after_state_id=after_id, - before_observation_id=before_observation_id, - after_observation_id=after_observation_id, - observed_changes=observed, - proposed_postconditions=proposed, - status=status, - confidence=confidence, - evidence={ - **(evidence or {}), - "locator_match_count": match_count, - "locator_unique": unique, - "primary_locator": primary_expr, - }, - raw_event=dict(clean_raw_event or {}), - raw_event_ids=list(raw_event_ids or []), - ) - write_model_json(self.actions_dir / f"{act.id}.json", act) - self._actions_by_id[act.id] = act - self.action_flow.actions.append(act.id) - for observation_id in (before_observation_id, after_observation_id): - if observation_id and observation_id not in self.action_flow.observations: - self.action_flow.observations.append(observation_id) - if after_id: - self.action_flow.final_state_id = after_id - self.last_action = act - self._persist_flow() - self._notify_action_update(act, "captured") - if self.on_action is not None and action_type != "checkpoint": - try: - self.on_action(act) - except Exception as exc: - self.last_warning = f"action evidence scheduling failed: {exc}" - # Dual-write optional pointer fields into flow.json via side file is enough; - # artifacts integration updates flow.json separately. - return act - - def attach_eventual_evidence( - self, - action_ids: list[str], - *, - dump: Any, - requested_monotonic: float, - ) -> None: - """Attach one coalesced full dump without rewriting causal after-state.""" - with self._lock: - lag_ms = max(0, int((time.monotonic() - requested_monotonic) * 1000)) - state = self.snapshots.capture_from_dump( - dump, change_reason="eventual_after_actions", force_persist=True - ) - observation = self.snapshots.latest_observation - if state.id not in self.action_flow.states: - self.action_flow.states.append(state.id) - if observation and observation.id not in self.action_flow.observations: - self.action_flow.observations.append(observation.id) - self.action_flow.final_state_id = state.id - if (self.action_flow.surface or "") == "windows": - self._promote_windows_targets_from_dump( - dump, action_ids=action_ids, evidence_lag_ms=lag_ms - ) - for action_id in action_ids: - action = self._actions_by_id.get(action_id) - if action is None: - continue - action.evidence.update( - { - "eventual_observation_id": observation.id if observation else "", - "eventual_state_id": state.id, - "evidence_association": "coalesced_eventual", - "evidence_lag_ms": lag_ms, - } - ) - write_model_json(self.actions_dir / f"{action.id}.json", action) - self._notify_action_update(action, "evidence") - self._persist_flow() - - @staticmethod - def _is_oversized_windows_target(target: dict[str, Any]) -> bool: - bounds = dict(target.get("bounds") or target.get("rect") or {}) - area = int(bounds.get("width") or 0) * int(bounds.get("height") or 0) - return str(target.get("control_type") or "") == "WindowControl" or area > 1_000_000 - - @staticmethod - def _is_broad_windows_target(target: dict[str, Any]) -> bool: - if ActionRecordingManager._is_oversized_windows_target(target): - return True - bounds = dict(target.get("bounds") or target.get("rect") or {}) - area = int(bounds.get("width") or 0) * int(bounds.get("height") or 0) - control_type = str(target.get("control_type") or "") - name = str(target.get("name") or target.get("automation_id") or "").lower() - layout_tokens = ( - "search_result", "listview", "list_view", "watermark", - "container", "layout", "clientbg", "panel", "pane", - ) - return ( - area > 250_000 - or ( - control_type in {"TreeControl", "PaneControl", "GroupControl"} - and any(token in name for token in layout_tokens) - ) - ) - - @staticmethod - def _unique_windows_row_candidate( - rows: list[dict[str, Any]], - ) -> Optional[dict[str, Any]]: - """Return a single visible result/list row when geometry is unreliable.""" - row_tokens = ( - "user_item", "result_item", "search_item", "contact_item", - "list_item", "listitem", "result_row", "contact_row", - ) - candidates: dict[tuple[Any, ...], tuple[int, dict[str, Any]]] = {} - for index, row in enumerate(rows): - if row.get("is_offscreen") is True: - continue - control_type = str(row.get("control_type") or "") - name = " ".join( - str(row.get(key) or "") - for key in ("name", "automation_id", "class_name") - ).lower() - if not ( - any(token in name for token in row_tokens) - or control_type in {"ListItemControl", "DataItemControl"} - ): - continue - bounds = dict(row.get("bounds") or row.get("rect") or {}) - width = int(bounds.get("width") or 0) - height = int(bounds.get("height") or 0) - if width < 8 or height < 8: - continue - key = ( - int(bounds.get("left") or 0), int(bounds.get("top") or 0), - int(bounds.get("right") or 0), int(bounds.get("bottom") or 0), - str(row.get("name") or ""), str(row.get("value") or ""), - ) - candidates[key] = (index, dict(row)) - if len(candidates) != 1: - return None - index, chosen = next(iter(candidates.values())) - chosen["bounds"] = dict(chosen.get("bounds") or chosen.get("rect") or {}) - chosen.pop("rect", None) - # A row container often has no accessible value of its own. Prefer an - # explicit row-name descendant over structural labels such as - # search_avatar_label, and strip WeCom rich-text markup. - row_depth = int(chosen.get("depth") or 0) - structural = { - "layout", "container", "controlui", "horizontal", "vertical", - "user_item", "list_item", "result_item", - } - descendants: list[tuple[int, str, str, dict[str, Any]]] = [] - preferred_tokens = ( - "search_name", "groupbuddyname", "contact_name", "display_name", - "result_name", "title_label", - ) - rejected_tokens = ("avatar", "layout", "container", "controlui") - for child in rows[index + 1 :]: - child_depth = int(child.get("depth") or 0) - if child_depth <= row_depth: - break - raw_value = str(child.get("value") or "").strip() - name = str(child.get("name") or "").strip() - clean_value = html.unescape(re.sub(r"<[^>]+>", "", raw_value)).strip() - lowered = name.lower() - semantic = clean_value or ( - name if name and not any(token in lowered for token in structural) else "" - ) - if not semantic or semantic == "[REDACTED]" or any( - token in lowered for token in rejected_tokens - ): - continue - rank = 0 if any(token in lowered for token in preferred_tokens) else 1 - descendants.append((rank, semantic, name, child)) - if descendants: - _rank, semantic, name, child = min(descendants, key=lambda item: item[0]) - chosen["semantic_name"] = semantic - chosen["row_descendant"] = { - "name": name, - "value": str(child.get("value") or "").strip(), - "control_type": child.get("control_type") or "", - } - if not str(chosen.get("value") or ""): - chosen["value"] = semantic - return chosen - - @staticmethod - def _windows_hit_candidate( - rows: list[dict[str, Any]], x: int, y: int - ) -> Optional[dict[str, Any]]: - interactive = { - "ButtonControl", "EditControl", "TreeItemControl", "ListItemControl", - "MenuItemControl", "CheckBoxControl", "RadioButtonControl", - "ComboBoxControl", "TabItemControl", "HyperlinkControl", - } - semantic_tokens = ( - "search", "searchedit", "edit", "input", "message", "send", - "comment", "reply", "搜索", "输入", "发送", "消息", - ) - candidates: list[tuple[tuple[int, int, int, int], dict[str, Any]]] = [] - for row in rows: - bounds = dict(row.get("bounds") or row.get("rect") or {}) - try: - inside = ( - int(bounds.get("left")) <= x <= int(bounds.get("right")) - and int(bounds.get("top")) <= y <= int(bounds.get("bottom")) - ) - except (TypeError, ValueError): - continue - if not inside or row.get("is_offscreen") is True: - continue - width = max(0, int(bounds.get("width") or 0)) - height = max(0, int(bounds.get("height") or 0)) - area = width * height - control_type = str(row.get("control_type") or "") - # Hairlines, separators and layout-only artifacts are never useful - # click targets even when they happen to be the smallest hit. - if ( - not area - or width < 4 - or height < 4 - or control_type == "WindowControl" - ): - continue - text = " ".join( - str(row.get(key) or "") - for key in ("name", "value", "automation_id", "class_name") - ).lower() - if any(token in text for token in ("separator", "seperator", "divider", "splitter")): - continue - # A named editor/result is stronger than a smaller generic list - # item covering the same point. - exact_editor = any( - token in text - for token in ("searchedit", "send_rich_edit", "compose_box", "search_box") - ) and "_layout" not in text - if exact_editor: - priority = 0 - elif any(token in text for token in semantic_tokens): - priority = 1 - elif control_type in interactive: - priority = 2 - else: - priority = 3 - layout_penalty = 1 if "layout" in text or "container" in text else 0 - redacted = 1 if "[redacted]" in text else 0 - candidates.append(((priority, layout_penalty, redacted, area), row)) - if not candidates: - return None - chosen = dict(min(candidates, key=lambda item: item[0])[1]) - bounds = dict(chosen.get("bounds") or chosen.get("rect") or {}) - chosen["bounds"] = bounds - chosen.pop("rect", None) - chosen["capture_point"] = {"x": x, "y": y} - return chosen - - @staticmethod - def _windows_candidate_quality(target: dict[str, Any]) -> tuple[int, int, int]: - bounds = dict(target.get("bounds") or target.get("rect") or {}) - area = int(bounds.get("width") or 0) * int(bounds.get("height") or 0) - control_type = str(target.get("control_type") or "") - text = " ".join( - str(target.get(key) or "") - for key in ("name", "value", "automation_id", "class_name") - ).lower() - interactive = control_type in { - "ButtonControl", "EditControl", "TreeItemControl", "ListItemControl", - "MenuItemControl", "CheckBoxControl", "RadioButtonControl", - "ComboBoxControl", "TabItemControl", "HyperlinkControl", - } - semantic = any( - token in text - for token in ("search", "edit", "input", "message", "send", "buddy", "result") - ) - exact_editor = any( - token in text - for token in ("searchedit", "send_rich_edit", "compose_box", "search_box") - ) and "_layout" not in text - return (2 if exact_editor else (1 if semantic else 0), 1 if interactive else 0, -area) - - @staticmethod - def _clean_windows_rich_text(value: str) -> str: - return html.unescape(re.sub(r"<[^>]+>", "", str(value or ""))).strip() - - @staticmethod - def _same_windows_target_scope( - first: dict[str, Any], second: dict[str, Any] - ) -> bool: - a = dict(first.get("window_scope") or {}) - b = dict(second.get("window_scope") or {}) - ah, bh = int(a.get("hwnd") or 0), int(b.get("hwnd") or 0) - if ah and bh: - return ah == bh - ap, bp = int(a.get("pid") or 0), int(b.get("pid") or 0) - ac = str(a.get("class_name") or first.get("class_name") or "") - bc = str(b.get("class_name") or second.get("class_name") or "") - return bool(ap and ap == bp and (not ac or not bc or ac == bc)) - - @staticmethod - def _windows_scope_key(target: dict[str, Any]) -> tuple[Any, ...]: - scope = dict(target.get("window_scope") or {}) - hwnd = int(scope.get("hwnd") or 0) - if hwnd: - return ("hwnd", hwnd) - return ( - "process", - int(scope.get("pid") or 0), - str(scope.get("class_name") or target.get("class_name") or ""), - ) - - @staticmethod - def _is_dynamic_windows_candidate(target: dict[str, Any]) -> bool: - name = str(target.get("name") or target.get("automation_id") or "").lower() - value = str(target.get("value") or "").strip() - dynamic_tokens = ( - "description", "preview", "snippet", "latest_message", - "last_message", "timestamp", "time_label", "消息摘要", "最新消息", - ) - return bool(value and any(token in name for token in dynamic_tokens)) - - @staticmethod - def _same_windows_scope(target: dict[str, Any], window: dict[str, Any]) -> bool: - scope = dict(target.get("window_scope") or {}) - target_hwnd = int(scope.get("hwnd") or 0) - dump_hwnd = int(window.get("hwnd") or 0) - if target_hwnd and dump_hwnd: - return target_hwnd == dump_hwnd - target_pid = int(scope.get("pid") or 0) - dump_pid = int(window.get("pid") or 0) - target_class = str(scope.get("class_name") or target.get("class_name") or "") - dump_class = str(window.get("class_name") or "") - if not scope: - # Synthetic/unit captures may only carry the root class. Real - # Windows events include hwnd/pid; class equality is the safest - # compatibility fallback when that identity is unavailable. - return bool(target_class and target_class == dump_class) - return bool(target_pid and target_pid == dump_pid and target_class == dump_class) - - def _promote_windows_targets_from_dump( - self, - dump: Any, - *, - action_ids: Optional[list[str]] = None, - evidence_lag_ms: int = 0, - ) -> None: - rows = list(getattr(dump, "locatable", None) or []) - window = dict(getattr(dump, "window", None) or {}) - if not rows: - return - last_business_by_scope: dict[tuple[Any, ...], dict[str, Any]] = {} - allowed = set(action_ids or []) - actions = sorted(self._actions_by_id.values(), key=lambda item: item.sequence) - for action in actions: - target = dict(action.target_snapshot or {}) - scope_key = self._windows_scope_key(target) - should_update = not allowed or action.id in allowed - if not should_update: - if target and not self._is_broad_windows_target(target): - last_business_by_scope[scope_key] = target - continue - # Eventual evidence from another application or from WeCom's main - # window must never rewrite a transient popup/screenshot action. - if not self._same_windows_scope(target, window): - continue - promoted: Optional[dict[str, Any]] = None - promotion_reason = "eventual_tree_coordinate_hit" - if action.action_type == "click" and self._is_broad_windows_target(target): - raw = dict(action.raw_event or {}) - meta = dict(raw.get("meta") or {}) - point = dict(meta.get("uia_coordinates") or {}) - points: list[tuple[int, int]] = [] - x = point.get("x", raw.get("x")) - y = point.get("y", raw.get("y")) - if x is not None and y is not None: - points.append((int(x), int(y))) - for candidate in meta.get("uia_coordinate_candidates") or []: - cx, cy = candidate.get("x"), candidate.get("y") - if cx is not None and cy is not None: - p = (int(cx), int(cy)) - if p not in points: - points.append(p) - hits = [ - hit - for px, py in points - for hit in [self._windows_hit_candidate(rows, px, py)] - if hit is not None - ] - if hits: - promoted = max(hits, key=self._windows_candidate_quality) - window_class = str(window.get("class_name") or "") - if ( - (not promoted or self._is_broad_windows_target(promoted)) - and window_class == "SearchResultWindow2" - ): - unique_row = self._unique_windows_row_candidate(rows) - if unique_row: - promoted = unique_row - promotion_reason = "eventual_tree_unique_row" - elif action.action_type in {"fill", "type", "press_key"} and self._is_oversized_windows_target(target): - promoted = dict(last_business_by_scope.get(scope_key) or {}) or None - elif target and not self._is_broad_windows_target(target): - last_business_by_scope[scope_key] = target - - if not promoted: - continue - if evidence_lag_ms > 5_000 and self._is_dynamic_windows_candidate(promoted): - action.evidence.update( - { - "target_promotion_rejected": "late_dynamic_candidate", - "rejected_candidate_name": promoted.get("name") or "", - "rejected_evidence_lag_ms": int(evidence_lag_ms), - } - ) - write_model_json(self.actions_dir / f"{action.id}.json", action) - continue - promoted["window_scope"] = dict( - promoted.get("window_scope") or target.get("window_scope") or window - ) - promoted_name = str(promoted.get("name") or "").lower() - if any( - token in promoted_name - for token in ("search_name", "contact_name", "groupbuddyname", "display_name") - ): - cleaned_value = self._clean_windows_rich_text( - str(promoted.get("value") or "") - ) - if cleaned_value: - promoted["value"] = cleaned_value - promoted["semantic_name"] = cleaned_value - element = self.catalog.upsert_from_snapshot( - promoted, - surface="windows", - window_scope=promoted.get("window_scope") or {}, - rows=rows, - ) - action.target_snapshot = sanitize_artifact(promoted) - action.target_element_id = element.id - action.confidence = max(float(action.confidence or 0.0), 0.72) - action.evidence.update( - { - "target_promotion": promotion_reason, - "original_target_control_type": target.get("control_type") or "", - "locator_match_count": ( - element.primary_locator.actual_count if element.primary_locator else None - ), - "locator_unique": ( - element.primary_locator.unique if element.primary_locator else None - ), - "primary_locator": ( - element.primary_locator.expression if element.primary_locator else "" - ), - } - ) - if isinstance(action.raw_event, dict): - action.raw_event["target"] = sanitize_artifact(promoted) - if element.id not in self.action_flow.elements: - self.action_flow.elements.append(element.id) - last_business_by_scope[scope_key] = promoted - write_model_json(self.actions_dir / f"{action.id}.json", action) - - def note_evidence_failure(self, action_ids: list[str], error: str) -> None: - with self._lock: - self.last_warning = f"eventual evidence failed: {error}" - for action_id in action_ids: - action = self._actions_by_id.get(action_id) - if action is None: - continue - action.evidence["eventual_evidence_error"] = str(error) - write_model_json(self.actions_dir / f"{action.id}.json", action) - self._notify_action_update(action, "evidence_failed") - - def _notify_action_update(self, action: ActionRecord, phase: str) -> None: - if self.on_action_update is None: - return - try: - self.on_action_update(action, phase) - except Exception as exc: - self.last_warning = f"action UI update failed: {exc}" - - def _persist_flow(self) -> None: - write_model_json(self.flow_dir / "action_flow.json", self.action_flow) - - def heartbeat(self) -> None: - """Lightweight monitor: flush idle input; do NOT create states/actions.""" - if self.status != RecorderState.RECORDING: - return - with self._lock: - for item in self.aggregator.maybe_flush_idle(): - if item.action_type == "press_key": - self._commit_press_key(item) - else: - self._commit_fill(item) - - def stats(self) -> dict[str, Any]: - la = self.last_action - return { - "recorder_status": self.status, - "state_count": len(self.action_flow.states), - "observation_count": len(self.snapshots.observation_ids()), - "action_count": len(self.action_flow.actions), - "element_count": len(self.catalog.catalog.elements), - "last_action_type": la.action_type if la else "", - "last_target": (la.target_snapshot or {}).get("name") - or (la.target_snapshot or {}).get("automation_id") - or (la.target_snapshot or {}).get("semantic_name") - or (la.target_snapshot or {}).get("class_name") - or (la.target_snapshot or {}).get("tag") - if la - else "", - "last_match_count": (la.evidence or {}).get("locator_match_count") if la else None, - "last_unique": (la.evidence or {}).get("locator_unique") if la else None, - "last_page_id": ((la.raw_event or {}).get("meta") or {}).get("page_id") if la else "", - "last_observation_id": ( - la.after_observation_id - or (la.evidence or {}).get("eventual_observation_id") - if la else "" - ), - "warning": self.last_warning, - "recording_mode": self.action_flow.recording_mode, - } diff --git a/tools/ui_inspector/recorder/windows.py b/tools/ui_inspector/recorder/windows.py deleted file mode 100644 index de1644d..0000000 --- a/tools/ui_inspector/recorder/windows.py +++ /dev/null @@ -1,1711 +0,0 @@ -"""Event-driven Windows action recorder. - -Input is observed out-of-process with low-level Win32 hooks. Hook callbacks do -only constant-time queue writes; UIA resolution happens on a separate worker so -a slow provider (WeCom/SAP/etc.) cannot make Windows remove the hook. The old -``GetAsyncKeyState`` polling path remains only as a mouse fallback when hook -installation is unavailable. -""" - -from __future__ import annotations - -import ctypes -import os -import queue -import threading -import time -from ctypes import wintypes -from typing import Any, Callable, Optional - -from .base import RawUserEvent, RecorderBase, RecorderState - -# Win32 virtual key codes -_VK_LBUTTON = 0x01 -_VK_RBUTTON = 0x02 -_VK_MBUTTON = 0x04 - -_WH_KEYBOARD_LL = 13 -_WH_MOUSE_LL = 14 -_WM_QUIT = 0x0012 -_WM_KEYDOWN = 0x0100 -_WM_SYSKEYDOWN = 0x0104 -_WM_LBUTTONDOWN = 0x0201 -_WM_LBUTTONUP = 0x0202 -_WM_RBUTTONDOWN = 0x0204 -_WM_RBUTTONUP = 0x0205 -_LLMHF_INJECTED = 0x00000001 - - -class _POINT(ctypes.Structure): - _fields_ = [("x", wintypes.LONG), ("y", wintypes.LONG)] - - -class _MSLLHOOKSTRUCT(ctypes.Structure): - _fields_ = [ - ("pt", _POINT), - ("mouseData", wintypes.DWORD), - ("flags", wintypes.DWORD), - ("time", wintypes.DWORD), - ("dwExtraInfo", ctypes.c_size_t), - ] - - -class _KBDLLHOOKSTRUCT(ctypes.Structure): - _fields_ = [ - ("vkCode", wintypes.DWORD), - ("scanCode", wintypes.DWORD), - ("flags", wintypes.DWORD), - ("time", wintypes.DWORD), - ("dwExtraInfo", ctypes.c_size_t), - ] - - -_SPECIAL_KEYS = { - 0x08: "Backspace", - 0x09: "Tab", - 0x0D: "Enter", - 0x1B: "Escape", - 0x21: "PageUp", - 0x22: "PageDown", - 0x23: "End", - 0x24: "Home", - 0x25: "ArrowLeft", - 0x26: "ArrowUp", - 0x27: "ArrowRight", - 0x28: "ArrowDown", - 0x2E: "Delete", - **{0x70 + i: f"F{i + 1}" for i in range(12)}, -} - - -class WindowsRecorder(RecorderBase): - """Observe Windows input and resolve UIA targets outside target processes.""" - - capability_map = { - "click": "implemented", - "fill": "implemented", - "press_key": "implemented", - "double_click": "schema_only", - "right_click": "partial", - } - - def __init__( - self, - *, - on_event: Optional[Callable[[RawUserEvent], None]] = None, - exclude_hwnds: Optional[Callable[[], set[int]]] = None, - poll_sec: float = 0.03, - focus_value_poll_sec: float = 0.4, - ) -> None: - super().__init__(on_event=on_event, exclude_hwnds=exclude_hwnds) - self.poll_sec = poll_sec - self.focus_value_poll_sec = focus_value_poll_sec - self._prev_down = False - self._last_focus_value = "" - self._last_focus_fp = "" - self._pending_target: dict[str, Any] = {} - self._last_uia_point: Optional[tuple[int, int]] = None - self._last_raw_point: Optional[tuple[int, int]] = None - self._last_root_hwnd = 0 - self._last_actionable_target: dict[str, Any] = {} - # Text editors need their own sticky target. A generic "last - # actionable" target is unsafe here: selecting a contact immediately - # before typing would otherwise make the contact row the input target. - self._last_text_entry_target: dict[str, Any] = {} - self._live_text_control: Any = None - self._element_atlas: dict[str, list[dict[str, Any]]] = {} - self._atlas_lock = threading.RLock() - self._ime_composing = False - self._last_typing_activity_identity = "" - self._last_typing_activity_at = 0.0 - self._input_queue: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=1024) - self._capture_thread: Optional[threading.Thread] = None - self._uia_thread: Optional[threading.Thread] = None - self._uia_request = threading.Event() - self._uia_finished = threading.Event() - self._snapshot_lock = threading.RLock() - self._latest_focus_snapshot: dict[str, Any] = {} - self._latest_focus_sampled_at = 0.0 - self._text_target_generation = 0 - self._uia_bound_target_generation = -1 - self._typing_active = False - self._last_key_at = 0.0 - self._accept_input = True - self._startup_ready = threading.Event() - self._hook_thread_id = 0 - self._mouse_hook = None - self._keyboard_hook = None - self._mouse_proc = None - self._keyboard_proc = None - self._health_lock = threading.Lock() - self._health: dict[str, Any] = { - "backend": "starting", - "mouse_hook": False, - "keyboard_hook": False, - "raw_input_events": 0, - "mouse_events": 0, - "keyboard_events": 0, - "resolved_events": 0, - "queue_dropped": 0, - "loop_exceptions": 0, - "last_error": "", - "last_event_monotonic": None, - "undrained_events": 0, - "shutdown_complete": True, - "uia_samples": 0, - } - - @property - def capability(self) -> dict[str, str]: - return dict(self.capability_map) - - def start(self) -> None: - if self._thread and self._thread.is_alive(): - self.status = RecorderState.RECORDING - return - self._stop.clear() - self._startup_ready.clear() - self._uia_finished.clear() - self._uia_request.clear() - self._accept_input = True - self._health_set(shutdown_complete=False, undrained_events=0) - self.status = RecorderState.RECORDING - self._capture_thread = threading.Thread( - target=self._capture_loop, - name="ui-inspector-win-input-consumer", - daemon=True, - ) - self._capture_thread.start() - self._uia_thread = threading.Thread( - target=self._uia_sampler_loop, - name="ui-inspector-win-uia-sampler", - daemon=True, - ) - self._uia_thread.start() - self._thread = threading.Thread( - target=self._loop, name="ui-inspector-win-recorder", daemon=True - ) - self._thread.start() - if not self._startup_ready.wait(timeout=2.0): - self._record_error("Windows input backend did not complete startup handshake") - - def stop(self) -> None: - # Stop accepting hook packets first. The consumer then drains every - # packet already accepted while the independent UIA sampler performs - # one final read. Never report a clean shutdown with a hidden tail. - self._accept_input = False - self._stop.set() - self._uia_request.set() - if self._hook_thread_id: - try: - ctypes.windll.user32.PostThreadMessageW( - int(self._hook_thread_id), _WM_QUIT, 0, 0 - ) - except Exception as exc: - self._record_error(exc) - if self._thread and self._thread.is_alive(): - self._thread.join(timeout=2.0) - listener_alive = bool(self._thread and self._thread.is_alive()) - if not listener_alive: - self._thread = None - if self._uia_thread and self._uia_thread.is_alive(): - self._uia_thread.join(timeout=3.0) - uia_alive = bool(self._uia_thread and self._uia_thread.is_alive()) - if not uia_alive: - self._uia_thread = None - if self._capture_thread and self._capture_thread.is_alive(): - self._capture_thread.join(timeout=5.0) - resolver_alive = bool(self._capture_thread and self._capture_thread.is_alive()) - if not resolver_alive: - self._capture_thread = None - undrained = int(self._input_queue.qsize()) - complete = not listener_alive and not resolver_alive and undrained == 0 - self._health_set( - undrained_events=undrained, - shutdown_complete=complete, - uia_sampler_still_running=uia_alive, - ) - self.status = RecorderState.STOPPED - if not complete: - raise RuntimeError( - "desktop recorder did not shut down cleanly: " - f"undrained={undrained}, listener_alive={listener_alive}, " - f"consumer_alive={resolver_alive}" - ) - - def health(self) -> dict[str, Any]: - with self._health_lock: - result = dict(self._health) - result["listener_thread_alive"] = bool( - self._thread and self._thread.is_alive() - ) - result["resolver_thread_alive"] = bool( - self._capture_thread and self._capture_thread.is_alive() - ) - result["uia_thread_alive"] = bool( - self._uia_thread and self._uia_thread.is_alive() - ) - result["input_queue_size"] = self._input_queue.qsize() - return result - - def _health_add(self, key: str, amount: int = 1) -> None: - with self._health_lock: - self._health[key] = int(self._health.get(key) or 0) + amount - - def _health_set(self, **values: Any) -> None: - with self._health_lock: - self._health.update(values) - - def _record_error(self, exc: Any) -> None: - self._health_add("loop_exceptions") - self._health_set(last_error=f"{type(exc).__name__}: {exc}" if isinstance(exc, BaseException) else str(exc)) - - def _loop(self) -> None: - if os.name != "nt": - self._health_set(backend="unsupported") - self._startup_ready.set() - return - try: - if self._run_hook_message_loop(): - return - except Exception as exc: - self._record_error(exc) - # Compatibility path: never the preferred backend. - self._health_set(backend="get_async_key_state_fallback") - self._startup_ready.set() - self._poll_mouse_fallback() - - def _run_hook_message_loop(self) -> bool: - user32 = ctypes.windll.user32 - kernel32 = ctypes.windll.kernel32 - lresult = ctypes.c_ssize_t - hook_proc_type = ctypes.WINFUNCTYPE( - lresult, ctypes.c_int, ctypes.c_size_t, ctypes.c_ssize_t - ) - user32.SetWindowsHookExW.argtypes = [ - ctypes.c_int, hook_proc_type, ctypes.c_void_p, wintypes.DWORD - ] - user32.SetWindowsHookExW.restype = ctypes.c_void_p - user32.CallNextHookEx.argtypes = [ - ctypes.c_void_p, ctypes.c_int, ctypes.c_size_t, ctypes.c_ssize_t - ] - user32.CallNextHookEx.restype = lresult - user32.UnhookWindowsHookEx.argtypes = [ctypes.c_void_p] - user32.UnhookWindowsHookEx.restype = wintypes.BOOL - kernel32.GetModuleHandleW.restype = ctypes.c_void_p - - @hook_proc_type - def mouse_proc(code, wparam, lparam): - if code >= 0 and wparam in { - _WM_LBUTTONDOWN, _WM_LBUTTONUP, _WM_RBUTTONDOWN, _WM_RBUTTONUP - }: - data = ctypes.cast( - lparam, ctypes.POINTER(_MSLLHOOKSTRUCT) - ).contents - self._enqueue_input( - { - "kind": "mouse", - "message": int(wparam), - "x": int(data.pt.x), - "y": int(data.pt.y), - "injected": bool(data.flags & _LLMHF_INJECTED), - "source_time_ms": int(data.time), - "coordinate_space": "physical_screen", - } - ) - return user32.CallNextHookEx(None, code, wparam, lparam) - - @hook_proc_type - def keyboard_proc(code, wparam, lparam): - if code >= 0 and wparam in {_WM_KEYDOWN, _WM_SYSKEYDOWN}: - data = ctypes.cast( - lparam, ctypes.POINTER(_KBDLLHOOKSTRUCT) - ).contents - self._enqueue_input( - { - "kind": "key", - "vk": int(data.vkCode), - "scan": int(data.scanCode), - "injected": bool(data.flags & 0x10), - "source_time_ms": int(data.time), - } - ) - return user32.CallNextHookEx(None, code, wparam, lparam) - - # Keep callbacks alive for the complete lifetime of the hooks. - self._mouse_proc = mouse_proc - self._keyboard_proc = keyboard_proc - module = kernel32.GetModuleHandleW(None) - self._hook_thread_id = int(kernel32.GetCurrentThreadId()) - self._mouse_hook = user32.SetWindowsHookExW( - _WH_MOUSE_LL, mouse_proc, module, 0 - ) - self._keyboard_hook = user32.SetWindowsHookExW( - _WH_KEYBOARD_LL, keyboard_proc, module, 0 - ) - mouse_ok = bool(self._mouse_hook) - keyboard_ok = bool(self._keyboard_hook) - self._health_set( - backend="low_level_hooks" if mouse_ok else "hook_failed", - mouse_hook=mouse_ok, - keyboard_hook=keyboard_ok, - ) - self._startup_ready.set() - if not mouse_ok: - if keyboard_ok: - user32.UnhookWindowsHookEx(self._keyboard_hook) - self._keyboard_hook = None - return False - - msg = wintypes.MSG() - try: - while not self._stop.is_set(): - result = int(user32.GetMessageW(ctypes.byref(msg), None, 0, 0)) - if result <= 0: - if result < 0: - self._record_error("GetMessageW failed") - break - user32.TranslateMessage(ctypes.byref(msg)) - user32.DispatchMessageW(ctypes.byref(msg)) - finally: - if self._mouse_hook: - user32.UnhookWindowsHookEx(self._mouse_hook) - if self._keyboard_hook: - user32.UnhookWindowsHookEx(self._keyboard_hook) - self._mouse_hook = None - self._keyboard_hook = None - self._hook_thread_id = 0 - return True - - def _poll_mouse_fallback(self) -> None: - user32 = ctypes.windll.user32 - while not self._stop.is_set(): - if self.status != RecorderState.RECORDING: - time.sleep(0.05) - continue - try: - down = bool(user32.GetAsyncKeyState(_VK_LBUTTON) & 0x8000) - if down and not self._prev_down: - x, y = self._cursor_pos(user32) - self._enqueue_input( - {"kind": "mouse", "message": _WM_LBUTTONDOWN, "x": x, "y": y, "fallback": True, "coordinate_space": "uia_screen"} - ) - elif not down and self._prev_down: - self._enqueue_input({"kind": "mouse", "message": _WM_LBUTTONUP, "fallback": True}) - self._prev_down = down - except Exception as exc: - self._record_error(exc) - time.sleep(self.poll_sec) - - def _enqueue_input(self, packet: dict[str, Any]) -> None: - if not self._accept_input: - self._health_add("input_rejected_after_stop") - return - self._health_add("raw_input_events") - self._health_add("mouse_events" if packet.get("kind") == "mouse" else "keyboard_events") - self._health_set(last_event_monotonic=time.monotonic()) - try: - self._input_queue.put_nowait(packet) - except queue.Full: - self._health_add("queue_dropped") - - def _capture_loop(self) -> None: - # This is deliberately a UIA-free path. Its only job is to preserve - # input order and turn cached snapshots into semantic events quickly. - while ( - not self._stop.is_set() - or not self._input_queue.empty() - or not self._uia_finished.is_set() - ): - packet = None - try: - packet = self._input_queue.get(timeout=0.05) - except queue.Empty: - pass - try: - if packet is not None and self.status == RecorderState.RECORDING: - self._dispatch_input_packet(packet) - self._health_add("resolved_events") - if ( - self._typing_active - and time.monotonic() - self._last_key_at >= 0.25 - ): - self._emit_cached_value_change(force=False) - except Exception as exc: - self._record_error(exc) - finally: - if packet is not None: - self._input_queue.task_done() - self._emit_cached_value_change(force=True) - - def _uia_sampler_loop(self) -> None: - """Own all UIA/COM objects and publish plain-dict snapshots only.""" - try: - import desktop_uia as uia - uia.ensure_com_sta() - except Exception as exc: - self._record_error(exc) - try: - while True: - self._uia_request.wait(timeout=self.focus_value_poll_sec) - self._uia_request.clear() - try: - with self._snapshot_lock: - generation = self._text_target_generation - requested_target = dict(self._last_text_entry_target or {}) - if ( - requested_target - and generation != self._uia_bound_target_generation - ): - self._bind_text_target(requested_target, resolve_live=True) - self._uia_bound_target_generation = generation - # Once an editor has been bound, read it directly even if a - # later click has moved OS focus. This preserves the final - # committed value at transaction boundaries. - live_info = self._snapshot_live_text_control() - info = self._focused_control_snapshot() - if live_info and self._typing_active: - info = live_info - if info: - self._publish_focus_snapshot(info) - self._health_add("uia_samples") - except Exception as exc: - self._record_error(exc) - if self._stop.is_set(): - break - finally: - self._uia_finished.set() - - def _publish_focus_snapshot(self, info: dict[str, Any]) -> None: - """Cross the UIA thread boundary using data, never COM controls.""" - if not info or self._is_transient_input_target(info): - return - snap = dict(info) - with self._snapshot_lock: - self._latest_focus_snapshot = snap - self._latest_focus_sampled_at = time.monotonic() - if self._is_text_entry(snap): - self._last_text_entry_target = dict(snap) - - def _cached_focus_snapshot(self) -> dict[str, Any]: - with self._snapshot_lock: - return dict(self._latest_focus_snapshot or {}) - - @staticmethod - def _focus_fingerprint(info: dict[str, Any]) -> str: - scope = dict(info.get("window_scope") or {}) - return "|".join( - [ - str(scope.get("hwnd") or 0), - str(info.get("automation_id") or ""), - str(info.get("name") or ""), - str(info.get("control_type") or ""), - ] - ) - - def _emit_cached_value_change(self, *, force: bool) -> None: - info = self._cached_focus_snapshot() - if not info or not self._is_text_entry(info): - return - fp = self._focus_fingerprint(info) - value = str(info.get("value") or "") - if fp != self._last_focus_fp: - # A late first sample may already contain the completed text. If - # keys were observed, preserve it instead of mistaking it for the - # baseline of a newly focused editor. - if not (force and self._typing_active and value): - self._last_focus_fp = fp - self._last_focus_value = value - return - if value == self._last_focus_value: - return - self.emit( - RawUserEvent( - kind="value_change", - surface="windows", - text=value, - target=info, - meta={ - "input_type": "password" - if "password" in str(info.get("name") or "").lower() - else "", - "sampled_at": self._latest_focus_sampled_at, - "causal_flush": bool(force), - }, - ) - ) - self._last_focus_fp = fp - self._last_focus_value = value - self._typing_active = False - - def _dispatch_input_packet(self, packet: dict[str, Any]) -> None: - message = int(packet.get("message") or 0) - if packet.get("kind") == "mouse": - if message in {_WM_LBUTTONDOWN, _WM_RBUTTONDOWN}: - self._on_mouse_down_at_fast( - int(packet.get("x") or 0), - int(packet.get("y") or 0), - button="right" if message == _WM_RBUTTONDOWN else "left", - source_meta=packet, - ) - elif message in {_WM_LBUTTONUP, _WM_RBUTTONUP}: - self._on_mouse_up() - elif packet.get("kind") == "key": - self._on_key_down(packet) - - def _on_mouse_down(self, user32) -> None: - point = self._cursor_pos(user32) - self._on_mouse_down_at_fast(*point) - - def _on_mouse_down_at_fast( - self, - x: int, - y: int, - *, - button: str = "left", - source_meta: Optional[dict[str, Any]] = None, - ) -> None: - """Resolve a click without touching UIA. - - Win32 supplies application ownership and the persisted element atlas - supplies an immediate semantic target. The sampler is then nudged to - refine focus/value state asynchronously. - """ - self._emit_cached_value_change(force=True) - raw_x, raw_y = int(x), int(y) - user32 = ctypes.windll.user32 - exclude: set[int] = set() - try: - exclude = set(self.exclude_hwnds() or set()) - except Exception: - pass - hwnd = 0 - root = 0 - pid = 0 - try: - point = self._make_point(raw_x, raw_y) - physical_lookup = getattr(user32, "WindowFromPhysicalPoint", None) - if physical_lookup: - physical_lookup.restype = ctypes.c_void_p - hwnd = int(physical_lookup(point) or 0) - else: - user32.WindowFromPoint.restype = ctypes.c_void_p - hwnd = int(user32.WindowFromPoint(point) or 0) - root = int(user32.GetAncestor(hwnd, 2) or hwnd) if hwnd else 0 - pid_ref = ctypes.c_ulong() - if root: - user32.GetWindowThreadProcessId(root, ctypes.byref(pid_ref)) - pid = int(pid_ref.value or 0) - if root in exclude or hwnd in exclude or pid == os.getpid(): - return - overlay = self._top_level_window_at_point( - raw_x, raw_y, exclude_hwnds=exclude, preferred_pid=pid - ) - if overlay and overlay != root: - root = int(overlay) - pid_ref = ctypes.c_ulong() - user32.GetWindowThreadProcessId(root, ctypes.byref(pid_ref)) - pid = int(pid_ref.value or 0) - except Exception as exc: - self._record_error(exc) - - scope = self._window_scope_fast(root or hwnd, fallback_pid=pid) - points = [(raw_x, raw_y)] - scale = 1.0 - try: - get_dpi = getattr(user32, "GetDpiForWindow", None) - dpi = int(get_dpi(int(root or hwnd)) or 96) if get_dpi and (root or hwnd) else 96 - scale = max(1.0, dpi / 96.0) - if scale > 1.05: - points.append((round(raw_x / scale), round(raw_y / scale))) - except Exception: - pass - target = self._atlas_hit(str(scope.get("class_name") or ""), points) - if target: - target["window_scope"] = dict(scope) - selection = "persisted_element_atlas" - else: - target = { - "name": str(scope.get("title") or ""), - "control_type": "WindowControl", - "class_name": str(scope.get("class_name") or ""), - "window_scope": dict(scope), - } - selection = "win32_window_fallback" - - self._last_raw_point = (raw_x, raw_y) - # Atlas coordinates are commonly logical on mixed-DPI desktop apps. - self._last_uia_point = points[-1] - self._last_root_hwnd = int(root or hwnd or 0) - self._pending_target = dict(target) - if not self._is_root_or_oversized(target): - self._last_actionable_target = dict(target) - if self._is_text_entry(target): - self._set_text_target_fast(target) - self.emit( - RawUserEvent( - kind="click", - surface="windows", - button=button, - x=raw_x, - y=raw_y, - target=target, - meta={ - "input_backend": self.health().get("backend"), - "injected": bool((source_meta or {}).get("injected")), - "source_time_ms": (source_meta or {}).get("source_time_ms"), - "source_coordinate_space": str( - (source_meta or {}).get("coordinate_space") or "uia_screen" - ), - "raw_coordinates_physical": {"x": raw_x, "y": raw_y}, - "coordinate_selection": selection, - "dpi_scale": scale, - "uia_deferred": True, - }, - ) - ) - self._uia_request.set() - - @staticmethod - def _window_scope_fast(hwnd: int, *, fallback_pid: int = 0) -> dict[str, Any]: - scope = {"hwnd": int(hwnd or 0), "pid": int(fallback_pid or 0), "title": "", "class_name": ""} - if not hwnd: - return scope - try: - user32 = ctypes.windll.user32 - title = ctypes.create_unicode_buffer(512) - klass = ctypes.create_unicode_buffer(256) - user32.GetWindowTextW(int(hwnd), title, len(title)) - user32.GetClassNameW(int(hwnd), klass, len(klass)) - pid_ref = ctypes.c_ulong() - user32.GetWindowThreadProcessId(int(hwnd), ctypes.byref(pid_ref)) - scope.update( - title=title.value, - class_name=klass.value, - pid=int(pid_ref.value or fallback_pid or 0), - ) - except Exception: - pass - return scope - - def _set_text_target_fast(self, info: dict[str, Any]) -> None: - with self._snapshot_lock: - self._last_text_entry_target = dict(info) - self._text_target_generation += 1 - self._last_focus_fp = self._focus_fingerprint(info) - self._last_focus_value = str(info.get("value") or "") - with self._snapshot_lock: - # The atlas target is useful immediately. The sampler will replace - # it with a live value-bearing snapshot when the provider responds. - self._latest_focus_snapshot = dict(info) - self._latest_focus_sampled_at = time.monotonic() - self._typing_active = False - self._uia_request.set() - - def _on_mouse_down_at( - self, - x: int, - y: int, - *, - button: str = "left", - source_meta: Optional[dict[str, Any]] = None, - ) -> None: - user32 = ctypes.windll.user32 - # Preserve the final value of the previously focused editor before the - # click changes focus (important for paste-then-click workflows). - if self._last_focus_fp: - self._poll_focus_value() - raw_x, raw_y = int(x), int(y) - exclude = set() - try: - exclude = set(self.exclude_hwnds() or set()) - except Exception: - exclude = set() - - # Filter Inspector window - root = 0 - try: - point = self._make_point(raw_x, raw_y) - physical_lookup = getattr(user32, "WindowFromPhysicalPoint", None) - user32.WindowFromPoint.restype = ctypes.c_void_p - if physical_lookup: - physical_lookup.restype = ctypes.c_void_p - hwnd = int( - (physical_lookup(point) if physical_lookup else user32.WindowFromPoint(point)) - or 0 - ) - root = int(user32.GetAncestor(hwnd, 2) or hwnd) if hwnd else 0 # GA_ROOT=2 - target_pid = ctypes.c_ulong() - if root: - user32.GetWindowThreadProcessId(root, ctypes.byref(target_pid)) - if ( - root in exclude - or hwnd in exclude - or int(target_pid.value or 0) == os.getpid() - ): - self.emit( - RawUserEvent( - kind="mouse_down", - surface="windows", - x=x, - y=y, - from_inspector=True, - meta={"reason": "inspector_hwnd"}, - ) - ) - return - overlay = self._top_level_window_at_point( - raw_x, - raw_y, - exclude_hwnds=exclude, - preferred_pid=int(target_pid.value or 0), - ) - if overlay and overlay != root: - root = overlay - target_pid = ctypes.c_ulong() - user32.GetWindowThreadProcessId(root, ctypes.byref(target_pid)) - except Exception: - hwnd = 0 - - uia_x, uia_y, dpi_meta = self._uia_point_for_window( - root or hwnd, raw_x, raw_y, - coordinate_space=str((source_meta or {}).get("coordinate_space") or "uia_screen"), - ) - target, err = self._element_from_point(uia_x, uia_y) - target_scope_hwnd = int((target.get("window_scope") or {}).get("hwnd") or 0) - target_scope_root = ( - int(user32.GetAncestor(target_scope_hwnd, 2) or target_scope_hwnd) - if target_scope_hwnd - else 0 - ) - if root and target_scope_root and target_scope_root != int(root): - root_target, root_error = self._element_from_handle(int(root)) - if root_target: - target = root_target - err = root_error - target, uia_x, uia_y, probe_meta = self._probe_mixed_uia_coordinates( - root or hwnd, - raw_x, - raw_y, - current_point=(uia_x, uia_y), - current_target=target or {}, - ) - dpi_meta.update(probe_meta) - target_scope = dict((target or {}).get("window_scope") or {}) - atlas_class = str(target_scope.get("class_name") or "") - if self._is_root_or_oversized(target or {}) and atlas_class: - atlas_points = [(int(uia_x), int(uia_y)), (raw_x, raw_y)] - for candidate in probe_meta.get("uia_coordinate_candidates") or []: - if candidate.get("x") is not None and candidate.get("y") is not None: - atlas_points.append((int(candidate["x"]), int(candidate["y"]))) - atlas_target = self._atlas_hit(atlas_class, atlas_points) - if atlas_target: - atlas_target["window_scope"] = dict(target_scope) - target = atlas_target - dpi_meta["coordinate_selection"] = "persisted_element_atlas" - dpi_meta["uia_coordinates"] = {"x": uia_x, "y": uia_y} - self._last_uia_point = (uia_x, uia_y) - self._last_raw_point = (raw_x, raw_y) - self._last_root_hwnd = int(root or hwnd or 0) - self._pending_target = target or {} - if self._pending_target and not self._is_root_or_oversized(self._pending_target): - self._last_actionable_target = dict(self._pending_target) - if self._is_text_entry(self._pending_target): - self._bind_text_target(self._pending_target) - self._seed_focus_baseline(self._pending_target) - meta: dict[str, Any] = { - "input_backend": self.health().get("backend"), - "injected": bool((source_meta or {}).get("injected")), - "source_time_ms": (source_meta or {}).get("source_time_ms"), - **dpi_meta, - } - if err: - meta["element_from_point_error"] = err - meta["raw_coordinates"] = {"x": x, "y": y} - # Coordinates saved as evidence only — never auto primary locator. - meta["coordinates_are_not_primary_locator"] = True - self.emit( - RawUserEvent( - kind="click", - surface="windows", - button=button, - x=uia_x, - y=uia_y, - target=self._pending_target, - meta=meta, - ) - ) - - @staticmethod - def _uia_point_for_window( - hwnd: int, - x: int, - y: int, - *, - coordinate_space: str, - ) -> tuple[int, int, dict[str, Any]]: - """Map WH_*_LL physical coordinates into the DPI-virtualized UIA space.""" - meta: dict[str, Any] = { - "raw_coordinates_physical": {"x": int(x), "y": int(y)}, - "source_coordinate_space": coordinate_space, - } - if coordinate_space != "physical_screen" or not hwnd: - meta["uia_coordinates"] = {"x": int(x), "y": int(y)} - return int(x), int(y), meta - try: - # UIAutomation providers do not necessarily use the same coordinate - # space as a DPI-aware WH_MOUSE_LL hook. In particular WeCom can - # expose a 1440x900 UIA root while the hook and DWM both report a - # 2880x1800 physical window. Comparing GetWindowRect with DWM alone - # therefore incorrectly produces a scale of 1. Prefer the actual - # UIA root rectangle as the destination coordinate space. - physical = wintypes.RECT() - hr = ctypes.windll.dwmapi.DwmGetWindowAttribute( - int(hwnd), 9, ctypes.byref(physical), ctypes.sizeof(physical) - ) - if hr == 0: - try: - import uiautomation as auto - - root = auto.ControlFromHandle(int(hwnd)) - rr = root.BoundingRectangle if root is not None else None - rw = int(rr.right - rr.left) if rr is not None else 0 - rh = int(rr.bottom - rr.top) if rr is not None else 0 - pw = int(physical.right - physical.left) - ph = int(physical.bottom - physical.top) - if min(rw, rh, pw, ph) > 0: - ux = round(int(rr.left) + (int(x) - physical.left) * rw / pw) - uy = round(int(rr.top) + (int(y) - physical.top) * rh / ph) - sx = pw / rw - sy = ph / rh - meta.update( - { - "uia_coordinates": {"x": ux, "y": uy}, - "dpi_scale": {"x": round(sx, 4), "y": round(sy, 4)}, - "coordinate_conversion": "dwm_physical_to_uia_root", - } - ) - return ux, uy, meta - except Exception as exc: - meta["uia_root_coordinate_error"] = f"{type(exc).__name__}: {exc}" - - logical = wintypes.RECT() - ctypes.windll.user32.GetWindowRect(int(hwnd), ctypes.byref(logical)) - # Compatibility fallback for providers whose root rectangle cannot - # be queried. DWMWA_EXTENDED_FRAME_BOUNDS is in physical pixels. - lw = int(logical.right - logical.left) - lh = int(logical.bottom - logical.top) - pw = int(physical.right - physical.left) - ph = int(physical.bottom - physical.top) - if hr == 0 and min(lw, lh, pw, ph) > 0: - sx = pw / lw - sy = ph / lh - ux = round(logical.left + (int(x) - physical.left) / sx) - uy = round(logical.top + (int(y) - physical.top) / sy) - meta.update( - { - "uia_coordinates": {"x": ux, "y": uy}, - "dpi_scale": {"x": round(sx, 4), "y": round(sy, 4)}, - "coordinate_conversion": "dwm_physical_to_getwindowrect_logical", - } - ) - return ux, uy, meta - except Exception as exc: - meta["coordinate_conversion_error"] = f"{type(exc).__name__}: {exc}" - meta["uia_coordinates"] = {"x": int(x), "y": int(y)} - meta["coordinate_conversion"] = "identity_fallback" - return int(x), int(y), meta - - @staticmethod - def _target_quality(target: dict[str, Any]) -> tuple[int, int, int, int, int]: - bounds = dict(target.get("bounds") or {}) - width = max(0, int(bounds.get("width") or 0)) - height = max(0, int(bounds.get("height") or 0)) - area = width * height - control_type = str(target.get("control_type") or "") - text = " ".join( - str(target.get(key) or "") - for key in ("name", "automation_id", "class_name", "control_type") - ).lower() - oversized = control_type == "WindowControl" or area > 1_000_000 - interactive = control_type in { - "ButtonControl", "EditControl", "TreeItemControl", "ListItemControl", - "MenuItemControl", "CheckBoxControl", "RadioButtonControl", - "ComboBoxControl", "TabItemControl", "HyperlinkControl", - } - semantic = any( - token in text - for token in ( - "search", "edit", "input", "message", "send", "contact", - "buddy", "result", "comment", "reply", - ) - ) - return ( - 0 if oversized else 1, - 1 if interactive else 0, - 1 if semantic else 0, - len(target.get("parent_chain") or []), - -area, - ) - - def _probe_mixed_uia_coordinates( - self, - hwnd: int, - raw_x: int, - raw_y: int, - *, - current_point: tuple[int, int], - current_target: dict[str, Any], - ) -> tuple[dict[str, Any], int, int, dict[str, Any]]: - """Resolve providers whose root and descendants use different DPI spaces. - - WeCom exposes a physical-size root but logical-size descendants. Only - probe alternative points when the direct hit is oversized and belongs - to a known mixed-space window, so standard desktop applications retain - their normal UIA hit-testing path. - """ - scope = dict(current_target.get("window_scope") or {}) - window_class = str(scope.get("class_name") or current_target.get("class_name") or "") - if ( - not self._is_root_or_oversized(current_target) - or window_class not in {"WeWorkWindow", "SearchResultWindow2"} - ): - return current_target, current_point[0], current_point[1], {} - - candidates: list[tuple[str, int, int, float]] = [ - ("direct", int(current_point[0]), int(current_point[1]), 1.0) - ] - try: - get_dpi = getattr(ctypes.windll.user32, "GetDpiForWindow", None) - dpi = int(get_dpi(int(hwnd)) or 96) if get_dpi else 96 - scale = max(1.0, float(dpi) / 96.0) - if scale > 1.05: - candidates.append( - ( - "window_dpi_absolute", - round(int(raw_x) / scale), - round(int(raw_y) / scale), - scale, - ) - ) - except Exception: - pass - - unique: list[tuple[str, int, int, float]] = [] - seen: set[tuple[int, int]] = set() - for item in candidates: - point = (item[1], item[2]) - if point not in seen: - seen.add(point) - unique.append(item) - - expected_pid = int(scope.get("pid") or 0) - expected_class = window_class - best_target = current_target - best_point = current_point - best_label = "direct" - best_scale = 1.0 - best_quality = self._target_quality(current_target) - attempts: list[dict[str, Any]] = [] - for label, px, py, scale in unique[1:]: - candidate, _err = self._element_from_point(px, py) - candidate_scope = dict((candidate or {}).get("window_scope") or {}) - same_scope = bool( - candidate - and (not expected_pid or int(candidate_scope.get("pid") or 0) == expected_pid) - and ( - not expected_class - or str(candidate_scope.get("class_name") or "") == expected_class - ) - ) - quality = self._target_quality(candidate or {}) if same_scope else (-1, 0, 0, 0, 0) - attempts.append( - { - "strategy": label, - "x": px, - "y": py, - "scale": round(scale, 4), - "target": str((candidate or {}).get("name") or ""), - "control_type": str((candidate or {}).get("control_type") or ""), - "same_scope": same_scope, - } - ) - if same_scope and quality > best_quality: - best_target = candidate - best_point = (px, py) - best_label = label - best_scale = scale - best_quality = quality - - return ( - best_target, - int(best_point[0]), - int(best_point[1]), - { - "uia_coordinate_candidates": attempts, - "coordinate_selection": best_label, - "mixed_coordinate_scale": round(best_scale, 4), - }, - ) - - def _on_key_down(self, packet: dict[str, Any]) -> None: - key = _SPECIAL_KEYS.get(int(packet.get("vk") or 0), "") - # Never call UIA here. Physical keys can arrive every few milliseconds; - # this consumer must remain faster than the hook producer even when the - # target accessibility provider is frozen for seconds. - target = self._cached_focus_snapshot() - if not self._is_text_entry(target): - fallback = dict(self._last_text_entry_target or {}) - if fallback and (not target or self._last_point_matches_target(fallback)): - target = fallback - if self._ime_composing: - # Enter/Escape inside an IME candidate/composition window confirms - # or cancels composition; it is not a business-level key action. - # Keep the transaction attached to the last real editor. - if target: - self._typing_active = True - self._last_key_at = time.monotonic() - self._emit_typing_activity(target, packet) - self._uia_request.set() - return - if not key: - self._typing_active = True - self._last_key_at = time.monotonic() - self._emit_typing_activity(target, packet) - self._uia_request.set() - return - if key in {"Backspace", "Delete"} and self._is_text_entry(target): - # Editing keys are part of the resulting value transaction. The - # next value poll records one complete fill instead of key spam. - self._typing_active = True - self._last_key_at = time.monotonic() - self._emit_typing_activity(target, packet) - self._uia_request.set() - return - # Enter/Tab/click are transaction boundaries: publish the latest - # sampled value before the boundary action, preserving causal order. - self._emit_cached_value_change(force=True) - self.emit( - RawUserEvent( - kind="key", - surface="windows", - key=key, - target=target, - meta={ - "input_backend": self.health().get("backend"), - "injected": bool(packet.get("injected")), - "source_time_ms": packet.get("source_time_ms"), - }, - ) - ) - self._uia_request.set() - - def _emit_typing_activity( - self, target: dict[str, Any], packet: dict[str, Any] - ) -> None: - """Journal one marker per text transaction without persisting key text. - - The marker lets diagnostics distinguish "no typing happened" from - "typing happened but the provider never exposed a value change". It - is intentionally not reduced into an Action. - """ - target = dict(target or self._last_text_entry_target or {}) - identity = "|".join( - str(target.get(key) or "") - for key in ("automation_id", "name", "control_type") - ) - scope = dict(target.get("window_scope") or {}) - identity = f"{scope.get('hwnd') or 0}|{identity}" - now = time.monotonic() - if ( - identity == self._last_typing_activity_identity - and now - self._last_typing_activity_at < 0.8 - ): - self._last_typing_activity_at = now - return - self._last_typing_activity_identity = identity - self._last_typing_activity_at = now - self.emit( - RawUserEvent( - kind="typing_activity", - surface="windows", - target=target, - meta={ - "input_backend": self.health().get("backend"), - "injected": bool(packet.get("injected")), - "source_time_ms": packet.get("source_time_ms"), - "content_recorded": False, - "unresolved_target": not bool(target), - "root_hwnd": int(self._last_root_hwnd or 0), - "last_uia_point": ( - {"x": self._last_uia_point[0], "y": self._last_uia_point[1]} - if self._last_uia_point else {} - ), - }, - ) - ) - - def _focused_control_snapshot(self) -> dict[str, Any]: - try: - import desktop_uia as uia - uia.ensure_com_sta() - import uiautomation as auto - ctrl = auto.GetFocusedControl() - info = self._control_snapshot(ctrl) if ctrl is not None else {} - self._ime_composing = self._is_transient_input_target(info) - unusable = self._is_root_or_oversized(info) or self._is_transient_input_target(info) - if unusable and self._last_uia_point: - try: - point_ctrl = auto.ControlFromPoint(*self._last_uia_point) - point_info = self._control_snapshot( - point_ctrl, x=self._last_uia_point[0], y=self._last_uia_point[1] - ) - if not self._is_root_or_oversized(point_info): - info = point_info - except Exception: - pass - if ( - (self._is_root_or_oversized(info) or self._is_transient_input_target(info)) - and self._last_raw_point - and self._last_uia_point - and self._last_root_hwnd - ): - try: - probed, _x, _y, _meta = self._probe_mixed_uia_coordinates( - self._last_root_hwnd, - self._last_raw_point[0], - self._last_raw_point[1], - current_point=self._last_uia_point, - current_target=info, - ) - if not self._is_root_or_oversized(probed) and not self._is_transient_input_target(probed): - info = probed - except Exception: - pass - if ( - (self._is_root_or_oversized(info) or self._is_transient_input_target(info)) - and self._last_text_entry_target - ): - refreshed = self._snapshot_live_text_control() - if refreshed: - info = refreshed - if ( - (self._is_root_or_oversized(info) or self._is_transient_input_target(info)) - and self._last_actionable_target - and self._last_point_matches_target(self._last_actionable_target) - ): - current_scope = dict(info.get("window_scope") or {}) - last_scope = dict(self._last_actionable_target.get("window_scope") or {}) - current_hwnd = int(current_scope.get("hwnd") or self._last_root_hwnd or 0) - last_hwnd = int(last_scope.get("hwnd") or 0) - if not current_hwnd or not last_hwnd or current_hwnd == last_hwnd: - info = dict(self._last_actionable_target) - if ( - (self._is_root_or_oversized(info) or self._is_transient_input_target(info)) - and self._last_uia_point - ): - scope = dict(info.get("window_scope") or {}) - class_name = str(scope.get("class_name") or "") - points = [self._last_uia_point] - if self._last_raw_point: - rx, ry = self._last_raw_point - points.extend( - [(rx, ry)] - + [(round(rx / scale), round(ry / scale)) for scale in (1.25, 1.5, 1.75, 2.0, 2.5, 3.0)] - ) - atlas_target = self._atlas_hit(class_name, points) - if atlas_target and self._is_text_entry(atlas_target): - atlas_target["window_scope"] = dict(scope) - self._bind_text_target(atlas_target) - info = self._snapshot_live_text_control() or atlas_target - if ( - (self._is_root_or_oversized(info) or self._is_transient_input_target(info)) - and self._last_text_entry_target - ): - current_scope = dict(info.get("window_scope") or {}) - editor_scope = dict(self._last_text_entry_target.get("window_scope") or {}) - current_pid = int(current_scope.get("pid") or 0) - editor_pid = int(editor_scope.get("pid") or 0) - # IME windows have a different HWND but normally share the - # application's PID. Never let the IME HWND replace the - # business editor just because the provider probe is cold. - if not current_pid or not editor_pid or current_pid == editor_pid: - refreshed = self._snapshot_live_text_control() - info = refreshed or dict(self._last_text_entry_target) - if info and not self._is_root_or_oversized(info) and not self._is_transient_input_target(info): - self._last_actionable_target = dict(info) - if self._is_text_entry(info): - self._bind_text_target(info, resolve_live=False) - return info - # A transient OS surface must never cross the recorder boundary as - # an application target. - return {} - except Exception as exc: - self._record_error(exc) - return {} - - def _on_mouse_up(self) -> None: - # Click already emitted on down (capture before UI disappears). - self._pending_target = {} - - def _poll_focus_value(self) -> None: - try: - info = self._focused_control_snapshot() - if not info: - return - # Skip if focused control belongs to Inspector - hwnd = int(info.get("window_scope", {}).get("hwnd") or 0) - try: - exclude = set(self.exclude_hwnds() or set()) - except Exception: - exclude = set() - if hwnd and hwnd in exclude: - return - fp = "|".join( - [ - str(info.get("automation_id") or ""), - str(info.get("name") or ""), - str(info.get("control_type") or ""), - ] - ) - val = str(info.get("value") or "") - if fp == self._last_focus_fp and val == self._last_focus_value: - return - if self._last_focus_fp and fp == self._last_focus_fp and val != self._last_focus_value: - self.emit( - RawUserEvent( - kind="value_change", - surface="windows", - text=val, - target=info, - meta={"input_type": "password" if "password" in (info.get("name") or "").lower() else ""}, - ) - ) - self._last_focus_fp = fp - self._last_focus_value = val - info["fingerprint"] = fp - except Exception: - return - - @staticmethod - def _is_text_entry(info: dict[str, Any]) -> bool: - text = " ".join( - str(info.get(key) or "") - for key in ("name", "automation_id", "class_name", "control_type") - ).lower() - return str(info.get("control_type") or "") == "EditControl" or any( - token in text - for token in ("edit", "input", "search", "compose", "message", "rich") - ) - - @staticmethod - def _is_transient_input_target(info: dict[str, Any]) -> bool: - text = " ".join( - str(value or "") - for value in ( - info.get("name"), - info.get("class_name"), - (info.get("window_scope") or {}).get("class_name"), - (info.get("window_scope") or {}).get("title"), - ) - ).lower() - return any( - token in text - for token in ( - "sopy_comp", "msctfime", "candidatewindow", "ime candidate", - "inputmethod", "input method", "compositionwindow", - "拼音", "候选词", "输入法", - ) - ) - - def _last_point_matches_target(self, target: dict[str, Any]) -> bool: - """Guard stale focus fallback with the last click's geometry.""" - bounds = dict(target.get("bounds") or {}) - try: - left = int(bounds.get("left")) - top = int(bounds.get("top")) - right = int(bounds.get("right")) - bottom = int(bounds.get("bottom")) - except (TypeError, ValueError): - return False - points: list[tuple[int, int]] = [] - if self._last_uia_point: - points.append(self._last_uia_point) - if self._last_raw_point: - rx, ry = self._last_raw_point - points.append((rx, ry)) - # Mixed-DPI providers commonly expose descendants at one of these - # Windows scale factors even when their root uses physical pixels. - for scale in (1.25, 1.5, 1.75, 2.0, 2.5, 3.0): - points.append((round(rx / scale), round(ry / scale))) - return any(left <= x <= right and top <= y <= bottom for x, y in points) - - def _seed_focus_baseline(self, info: dict[str, Any]) -> None: - """Remember the pre-input value at click time. - - Without this seed, fast typing can finish before the first 400 ms focus - poll and the completed text is mistaken for the initial baseline. - """ - if not info or not self._is_text_entry(info): - return - self._bind_text_target(info) - self._last_focus_fp = "|".join( - [ - str(info.get("automation_id") or ""), - str(info.get("name") or ""), - str(info.get("control_type") or ""), - ] - ) - self._last_focus_value = str(info.get("value") or "") - - def learn_elements( - self, - rows: list[dict[str, Any]], - *, - window: Optional[dict[str, Any]] = None, - ) -> None: - """Publish a bounded, value-free element atlas to the event-time path.""" - win = dict(window or {}) - class_name = str(win.get("class_name") or win.get("chosen_class") or "") - if not class_name: - return - cleaned: list[dict[str, Any]] = [] - for row in rows or []: - item = dict(row or {}) - bounds = dict(item.get("bounds") or item.get("rect") or {}) - if not bounds: - continue - item["bounds"] = bounds - item.pop("rect", None) - # Historical atlases are locators, not a store of user content. - item["value"] = "" - item["window_scope"] = { - "hwnd": int(win.get("hwnd") or 0), - "pid": int(win.get("pid") or 0), - "title": str(win.get("title") or ""), - "class_name": class_name, - } - cleaned.append(item) - if cleaned: - with self._atlas_lock: - self._element_atlas[class_name] = cleaned[-2500:] - - def _atlas_hit( - self, class_name: str, points: list[tuple[int, int]] - ) -> dict[str, Any]: - with self._atlas_lock: - rows = list(self._element_atlas.get(class_name) or []) - candidates: list[tuple[tuple[int, int, int, int], dict[str, Any]]] = [] - for row in rows: - bounds = dict(row.get("bounds") or {}) - try: - hit = any( - int(bounds.get("left")) <= x <= int(bounds.get("right")) - and int(bounds.get("top")) <= y <= int(bounds.get("bottom")) - for x, y in points - ) - except (TypeError, ValueError): - continue - if not hit or row.get("is_offscreen") is True: - continue - name = str(row.get("name") or row.get("automation_id") or "").lower() - control_type = str(row.get("control_type") or "") - width = max(0, int(bounds.get("width") or 0)) - height = max(0, int(bounds.get("height") or 0)) - area = width * height - if not area or width < 4 or height < 4: - continue - exact_editor = name in { - "searchedit", "send_rich_edit", "search_box", "compose_box" - } - editor = self._is_text_entry(row) and not name.endswith("_layout") - semantic = any( - token in name - for token in ("search_name", "contact_name", "send", "edit", "input") - ) - candidates.append( - ((1 if exact_editor else 0, 1 if editor else 0, 1 if semantic else 0, -area), row) - ) - return dict(max(candidates, default=((0, 0, 0, 0), {}), key=lambda x: x[0])[1]) - - def _bind_text_target( - self, info: dict[str, Any], *, resolve_live: bool = True - ) -> None: - if not info or not self._is_text_entry(info): - return - self._last_text_entry_target = dict(info) - if not resolve_live or not self._last_root_hwnd: - return - name = str(info.get("name") or "").strip() - if not name: - return - try: - import uiautomation as auto - - root = auto.ControlFromHandle(int(self._last_root_hwnd)) - candidate = root.Control(searchDepth=24, Name=name) - if candidate.Exists(0.35, 0.02): - self._live_text_control = candidate - except Exception: - self._live_text_control = None - - def _snapshot_live_text_control(self) -> dict[str, Any]: - if self._live_text_control is None: - return {} - try: - info = self._control_snapshot(self._live_text_control) - if info and self._is_text_entry(info): - return info - except Exception: - pass - self._live_text_control = None - return {} - - @staticmethod - def _is_root_or_oversized(info: dict[str, Any]) -> bool: - bounds = dict(info.get("bounds") or {}) - area = int(bounds.get("width") or 0) * int(bounds.get("height") or 0) - return str(info.get("control_type") or "") == "WindowControl" or area > 1_000_000 - - def _element_from_point(self, x: int, y: int) -> tuple[dict[str, Any], str]: - """Minimal directed read — NO ensure_ui_ready / full warm.""" - try: - import desktop_uia as uia - - uia.ensure_com_sta() - import uiautomation as auto - - ctrl = auto.ControlFromPoint(int(x), int(y)) - if ctrl is None: - return {}, "ControlFromPoint returned None" - return self._control_snapshot(ctrl, x=x, y=y), "" - except Exception as exc: - return {}, f"{type(exc).__name__}: {exc}" - - def _element_from_handle(self, hwnd: int) -> tuple[dict[str, Any], str]: - try: - import desktop_uia as uia - - uia.ensure_com_sta() - import uiautomation as auto - - ctrl = auto.ControlFromHandle(int(hwnd)) - if ctrl is None: - return {}, "ControlFromHandle returned None" - return self._control_snapshot(ctrl), "" - except Exception as exc: - return {}, f"{type(exc).__name__}: {exc}" - - @staticmethod - def _is_decorative_window_class(class_name: str) -> bool: - lowered = str(class_name or "").strip().lower() - return lowered in { - "shell_traywnd", "shell_secondarytraywnd", "progman", "workerw", - "ime", "msctfime ui", "tooltips_class32", "sopy_comp", - "perryshadowwnd", - } or any( - token in lowered - for token in ("shadowwnd", "dropshadow", "drop_shadow", "windowshadow") - ) - - @staticmethod - def _top_level_window_at_point( - x: int, - y: int, - *, - exclude_hwnds: set[int], - preferred_pid: int = 0, - ) -> int: - """Return the smallest visible top-level overlay containing a point. - - This is a generic Win32 fallback for applications whose accessibility - provider reports the owner window from ``WindowFromPoint`` while an - owned popup/list is visibly on top. - """ - if os.name != "nt": - return 0 - user32 = ctypes.windll.user32 - candidates: list[tuple[int, int]] = [] - @ctypes.WINFUNCTYPE(ctypes.c_bool, wintypes.HWND, wintypes.LPARAM) - def callback(hwnd, _lparam): - handle = int(hwnd or 0) - if not handle or handle in exclude_hwnds or not user32.IsWindowVisible(handle): - return True - pid_ref = ctypes.c_ulong() - user32.GetWindowThreadProcessId(handle, ctypes.byref(pid_ref)) - pid = int(pid_ref.value or 0) - if pid == os.getpid() or (preferred_pid and pid != preferred_pid): - return True - buf = ctypes.create_unicode_buffer(256) - user32.GetClassNameW(handle, buf, len(buf)) - class_name = buf.value - if WindowsRecorder._is_decorative_window_class(class_name): - return True - rect = wintypes.RECT() - if not user32.GetWindowRect(handle, ctypes.byref(rect)): - return True - if not (rect.left <= int(x) <= rect.right and rect.top <= int(y) <= rect.bottom): - return True - width = max(0, int(rect.right - rect.left)) - height = max(0, int(rect.bottom - rect.top)) - if width < 8 or height < 8: - return True - candidates.append((width * height, handle)) - return True - - try: - user32.EnumWindows(callback, 0) - except Exception: - return 0 - return min(candidates, default=(0, 0))[1] - - def _control_snapshot( - self, ctrl: Any, *, x: Optional[int] = None, y: Optional[int] = None - ) -> dict[str, Any]: - def safe(fn, default=""): - try: - return fn() - except Exception: - return default - - name = str(safe(lambda: ctrl.Name, "") or "") - value = "" - try: - value = str(ctrl.GetValuePattern().Value) - except Exception: - value = str(safe(lambda: getattr(ctrl, "Value", ""), "") or "") - if not value: - try: - value = str(ctrl.GetTextPattern().DocumentRange.GetText(-1) or "") - except Exception: - pass - if not value: - try: - value = str(ctrl.GetLegacyIAccessiblePattern().Value or "") - except Exception: - pass - aid = str(safe(lambda: ctrl.AutomationId, "") or "") - ctype = str(safe(lambda: ctrl.ControlTypeName, "") or "") - class_name = str(safe(lambda: ctrl.ClassName, "") or "") - framework = str(safe(lambda: ctrl.FrameworkId, "") or "") - enabled = safe(lambda: bool(ctrl.IsEnabled), None) - offscreen = safe(lambda: bool(ctrl.IsOffscreen), None) - rect: dict[str, int] = {} - try: - r = ctrl.BoundingRectangle - rect = { - "left": int(r.left), - "top": int(r.top), - "right": int(r.right), - "bottom": int(r.bottom), - "width": int(r.right - r.left), - "height": int(r.bottom - r.top), - } - except Exception: - rect = {} - - parent_chain: list[dict[str, str]] = [] - try: - p = ctrl.GetParentControl() - depth = 0 - while p is not None and depth < 8: - parent_chain.append( - { - "name": str(getattr(p, "Name", "") or ""), - "automation_id": str(getattr(p, "AutomationId", "") or ""), - "control_type": str(getattr(p, "ControlTypeName", "") or ""), - "class_name": str(getattr(p, "ClassName", "") or ""), - } - ) - p = p.GetParentControl() - depth += 1 - except Exception: - pass - - hwnd = 0 - pid = 0 - wtitle = "" - wclass = "" - try: - hwnd = int(ctrl.NativeWindowHandle or 0) - if not hwnd: - # walk up for hwnd - for item in parent_chain: - del item - p = ctrl - for _ in range(12): - h = int(getattr(p, "NativeWindowHandle", 0) or 0) - if h: - hwnd = h - break - p = p.GetParentControl() - if p is None: - break - if hwnd: - import desktop_uia as uia - - info = uia.window_info_from_hwnd(hwnd) if hasattr(uia, "window_info_from_hwnd") else None - if info is None: - # fallback via GetWindowText - import ctypes - - user32 = ctypes.windll.user32 - buf = ctypes.create_unicode_buffer(512) - user32.GetWindowTextW(hwnd, buf, 512) - wtitle = buf.value - user32.GetClassNameW(hwnd, buf, 512) - wclass = buf.value - pid_c = ctypes.c_ulong() - user32.GetWindowThreadProcessId(hwnd, ctypes.byref(pid_c)) - pid = int(pid_c.value) - else: - wtitle = info.title - wclass = info.class_name - pid = info.pid - hwnd = info.hwnd - except Exception: - pass - - snap = { - "name": name, - "value": value, - "automation_id": aid, - "control_type": ctype, - "class_name": class_name, - "framework_id": framework, - "is_enabled": enabled, - "is_offscreen": offscreen, - "bounds": rect, - "parent_chain": parent_chain, - "window_scope": { - "hwnd": hwnd, - "pid": pid, - "title": wtitle, - "class_name": wclass, - }, - } - if x is not None: - snap["capture_point"] = {"x": x, "y": y} - return snap - - @staticmethod - def _cursor_pos(user32) -> tuple[int, int]: - import ctypes - from ctypes import wintypes - - class POINT(ctypes.Structure): - _fields_ = [("x", wintypes.LONG), ("y", wintypes.LONG)] - - pt = POINT() - user32.GetCursorPos(ctypes.byref(pt)) - return int(pt.x), int(pt.y) - - @staticmethod - def _make_point(x: int, y: int): - import ctypes - from ctypes import wintypes - - class POINT(ctypes.Structure): - _fields_ = [("x", wintypes.LONG), ("y", wintypes.LONG)] - - return POINT(int(x), int(y)) diff --git a/tools/ui_inspector/redaction.py b/tools/ui_inspector/redaction.py deleted file mode 100644 index 7fe4f64..0000000 --- a/tools/ui_inspector/redaction.py +++ /dev/null @@ -1,206 +0,0 @@ -"""Sensitive-data redaction for Inspector artifacts.""" - -from __future__ import annotations - -import re -from typing import Any, Optional - -# Field / attribute name hints (case-insensitive). -_SENSITIVE_NAME_RE = re.compile( - r"(password|passwd|pwd|token|secret|cookie|authorization|auth|otp|" - r"captcha|verify.?code|verification.?code|sms.?code|pin)", - re.I, -) - -_PHONE_RE = re.compile(r"(? bool: - return bool(_SENSITIVE_NAME_RE.search(name or "")) - - -def looks_sensitive_value(value: str) -> bool: - s = value or "" - if not s: - return False - if _PHONE_RE.search(s) or _EMAIL_RE.search(s) or _TOKENISH_RE.fullmatch(s.strip()): - return True - if "password" in s.lower() or "secret" in s.lower(): - return True - return False - - -def sanitize_artifact(value: Any, *, sensitive_hint: bool = False) -> Any: - """Recursively sanitize event/evidence data before it is persisted. - - ``redact_input_payload`` protects the normalized Action input, but recorder - events also contain copies of the same value in target snapshots, ancestry - and raw_event. All persisted copies must pass through one boundary. - """ - if isinstance(value, dict): - input_type = str(value.get("input_type") or value.get("type") or "").lower() - field_name = str( - value.get("name") - or value.get("field") - or value.get("automation_id") - or "" - ) - local_sensitive = bool( - sensitive_hint - or input_type == "password" - or is_sensitive_field_name(field_name) - ) - out: dict[str, Any] = {} - for key, item in value.items(): - key_sensitive = local_sensitive or is_sensitive_field_name(str(key)) - if key in { - "name", "automation_id", "class_name", "control_type", - "framework_id", "path_hint", "logical_name", - } and isinstance(item, str) and not key_sensitive: - # Machine identifiers can be long token-like strings (for - # example conversation_list_container) without being secrets. - # Still remove explicit personal data embedded in a label. - out[key] = _EMAIL_RE.sub(REDACTED, _PHONE_RE.sub(REDACTED, item)) - continue - if key in {"value", "text", "input", "typed", "content", "plaintext"}: - if item in (None, ""): - out[key] = item - elif key_sensitive or looks_sensitive_value(str(item)): - out[key] = REDACTED - else: - out[key] = redact_text(str(item)) - else: - out[key] = sanitize_artifact(item, sensitive_hint=key_sensitive) - if local_sensitive: - out["sensitive"] = True - return out - if isinstance(value, list): - return [sanitize_artifact(item, sensitive_hint=sensitive_hint) for item in value] - if isinstance(value, tuple): - return [sanitize_artifact(item, sensitive_hint=sensitive_hint) for item in value] - if isinstance(value, str): - if sensitive_hint or looks_sensitive_value(value): - return REDACTED - return redact_text(value) - return value - - -def redact_text(value: str, *, force: bool = False) -> str: - if value is None: - return "" - s = str(value) - if force: - return REDACTED - s = _PHONE_RE.sub(REDACTED, s) - s = _EMAIL_RE.sub(REDACTED, s) - return s - - -def redact_input_payload( - payload: dict[str, Any], - *, - field_name: str = "", - input_type: str = "", -) -> tuple[dict[str, Any], bool]: - """ - Return (redacted_copy, was_sensitive). - Never leave raw password/token in action_flow / catalog. - """ - sensitive = False - out = dict(payload or {}) - itype = (input_type or out.get("input_type") or "").lower() - fname = field_name or str(out.get("name") or out.get("field") or "") - - if itype == "password" or is_sensitive_field_name(fname): - sensitive = True - for key in ("text", "value", "input", "typed", "content"): - if key in out and out[key] not in (None, ""): - out[key] = REDACTED - out["redacted"] = True - return out, True - - for key in ("text", "value", "input", "typed", "content"): - if key not in out: - continue - raw = out.get(key) - if raw is None: - continue - s = str(raw) - if looks_sensitive_value(s) or is_sensitive_field_name(str(out.get("name") or "")): - sensitive = True - force = ("@" in s) or bool(_PHONE_RE.search(s)) or bool(_TOKENISH_RE.search(s)) - out[key] = redact_text(s, force=force) - if _TOKENISH_RE.fullmatch(s.strip()) or looks_sensitive_value(s): - out[key] = REDACTED - if sensitive: - out["redacted"] = True - return out, sensitive - - -def redact_element_fields(el: dict[str, Any]) -> tuple[dict[str, Any], bool]: - out = dict(el or {}) - sensitive = False - name = str(out.get("name") or out.get("automation_id") or "") - role = str(out.get("control_type") or out.get("role") or "") - input_type = str(out.get("input_type") or out.get("type") or "") - if input_type == "password" or is_sensitive_field_name(name): - sensitive = True - if out.get("value"): - out["value"] = REDACTED - if out.get("text"): - out["text"] = REDACTED - out["sensitive"] = True - return out, True - for key in ("value", "text", "name"): - if key not in out or out[key] in (None, ""): - continue - s = str(out[key]) - # Stable element names/automation ids are locator structure. Long - # machine identifiers (OperationAndSendMessageRegion, generated DOM - # ids, etc.) must not be destroyed merely because they look token-like. - # Personal data embedded in a visible name is still scrubbed. - if key == "name": - cleaned = _EMAIL_RE.sub(REDACTED, _PHONE_RE.sub(REDACTED, s)) - out[key] = cleaned - sensitive = sensitive or cleaned != s - elif looks_sensitive_value(s): - sensitive = True - out[key] = REDACTED - if sensitive: - out["sensitive"] = True - # Edit controls with password-ish class - if "password" in role.lower() or "password" in str(out.get("class_name") or "").lower(): - sensitive = True - out["value"] = REDACTED - out["sensitive"] = True - return out, sensitive - - -def redact_locatable_rows(rows: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], bool]: - any_sens = False - out: list[dict[str, Any]] = [] - for r in rows or []: - cleaned, sens = redact_element_fields(r if isinstance(r, dict) else {}) - any_sens = any_sens or sens - out.append(cleaned) - return out, any_sens - - -def screenshot_sensitivity_meta( - *, - may_contain_sensitive: bool, - save_sensitive_screenshots: bool = False, -) -> dict[str, Any]: - return { - "may_contain_sensitive_data": bool(may_contain_sensitive), - "save_sensitive_screenshots": bool(save_sensitive_screenshots), - "note": ( - "截图暂时无法可靠脱敏；敏感页请关闭截图或人工审查。" - if may_contain_sensitive - else "" - ), - } diff --git a/tools/ui_inspector/requirements.txt b/tools/ui_inspector/requirements.txt deleted file mode 100644 index 2deb2b3..0000000 --- a/tools/ui_inspector/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -uiautomation>=2.0.18 -# 浏览器整页 dump（与技能共用宿主环境；本地开发可 editable 安装） -jiangchang-platform-kit>=1.2.11 -# playwright 由宿主 / kit 环境提供，勿在技能目录 playwright install diff --git a/tools/ui_inspector/run.py b/tools/ui_inspector/run.py deleted file mode 100644 index 9023b74..0000000 --- a/tools/ui_inspector/run.py +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python3 -"""Entry: python tools/ui_inspector/run.py [--out DIR] [--scene NAME]""" - -from __future__ import annotations - -import runpy -from pathlib import Path - -if __name__ == "__main__": - runpy.run_path(str(Path(__file__).resolve().parent / "app.py"), run_name="__main__") diff --git a/tools/ui_inspector/skip_policy.py b/tools/ui_inspector/skip_policy.py deleted file mode 100644 index 9848d1a..0000000 --- a/tools/ui_inspector/skip_policy.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Decide whether a dump is blank/empty-shell and should not become a flow step. - -Only applied to browser surface. Does NOT skip same-URL scrolls -(content may change without URL change). -""" - -from __future__ import annotations - -from typing import Any, Optional - -import target - -# Locatable below this counts as "no useful candidates" -_MIN_LOCATABLE = 3 -# Tree/index tiny enough to treat as empty shell -_MAX_EMPTY_NODES = 2 - - -def _norm_url(url: str) -> str: - return (url or "").strip() - - -def is_blankish_url(url: str) -> bool: - u = _norm_url(url).lower() - if not u: - return True - if u == "about:blank" or u.startswith("about:blank?"): - return True - if u in {"about:srcdoc", "about:newtab"}: - return True - if u.startswith("chrome://newtab") or u.startswith("chrome://new-tab-page"): - return True - if u.startswith("edge://newtab") or u.startswith("edge://new-tab-page"): - return True - return False - - -def should_skip_empty_dump( - *, - surface: str, - url: str = "", - locatable_count: Optional[int] = None, - nodes_kept: Optional[int] = None, - sparse: bool = False, - locatable: Optional[list[dict[str, Any]]] = None, - stats: Optional[dict[str, Any]] = None, -) -> Optional[str]: - """Return skip reason code, or None if the dump should be kept. - - Safe rules (browser only): - - blank/newtab URL **and** almost no locatable → skip - - tiny tree + sparse/blankish + almost no locatable → skip - - A page with about:blank URL but rich locatable (nav race) is **kept**. - """ - if surface != target.SURFACE_BROWSER: - return None - - st = stats or {} - if locatable_count is None: - if locatable is not None: - locatable_count = len(locatable) - else: - locatable_count = int(st.get("locatable_count") or 0) - if nodes_kept is None: - nodes_kept = int(st.get("nodes_kept") or st.get("findall_count") or 0) - if not sparse: - sparse = bool(st.get("uia_sparse")) - - loc_n = int(locatable_count or 0) - kept = int(nodes_kept or 0) - blankish = is_blankish_url(url) - - if blankish and loc_n < _MIN_LOCATABLE: - return "blank_or_newtab_empty" - - if loc_n < _MIN_LOCATABLE and kept <= _MAX_EMPTY_NODES and (sparse or blankish): - return "empty_shell" - - return None - - -def skip_reason_text(reason: str, *, url: str = "") -> str: - u = _norm_url(url) or "(empty)" - if reason == "blank_or_newtab_empty": - return f"跳过空页：url={u!r} 且可定位候选过少（blank/新标签页）。" - if reason == "empty_shell": - return f"跳过空壳页：url={u!r} 节点过少且 sparse。" - return f"跳过本拍：{reason} url={u!r}" diff --git a/tools/ui_inspector/smoke_check.py b/tools/ui_inspector/smoke_check.py deleted file mode 100644 index 134854f..0000000 --- a/tools/ui_inspector/smoke_check.py +++ /dev/null @@ -1,383 +0,0 @@ -"""Smoke test for flow document API + surface routing + kit helpers (v0.10).""" - -from __future__ import annotations - -import sys -import tempfile -from pathlib import Path - -_HERE = Path(__file__).resolve().parent -if str(_HERE) not in sys.path: - sys.path.insert(0, str(_HERE)) - -import artifacts as art -import desktop_uia as uia -import kit_browser -import kit_wake -import skip_policy -import target -import user_prefs -from surfaces.registry import get_driver -from drivers.mobile import MobileDriver -from jiangchang_skill_core.rpa.page_dump import build_page_dump_result - - -def _assert_selection_api() -> None: - assert callable(uia.window_info_under_cursor) - assert callable(uia.find_wecom_search_result_window) - assert callable(uia.resolve_capture_target) - try: - _info, _cands, meta = uia.resolve_capture_target( - under_cursor=False, - prefer_wecom_search_popup=False, - ) - assert "strategy" in meta - except RuntimeError as exc: - assert "未找到可拾取" in str(exc) - print("OK selection api") - - -def _assert_kit_wake_detect() -> None: - assert kit_wake.looks_like_wecom(class_name="WeWorkWindow", title="企业微信") - assert kit_wake.resolve_wake_profile(class_name="SearchResultWindow2") == "wecom_search" - assert not kit_wake.looks_like_wecom(class_name="Chrome_WidgetWin_1", title="Google") - r = kit_wake.auto_ensure_if_wecom(class_name="NotWeCom", title="x") - assert r.get("skipped") and r.get("reason") == "not_wecom" - print("OK kit_wake detect") - - -def _assert_surface_routing() -> None: - assert ( - target.resolve_surface(process_name="chrome.exe", class_name="Chrome_WidgetWin_1") - == target.SURFACE_BROWSER - ) - assert ( - target.resolve_surface(process_name="msedge.exe", class_name="Chrome_WidgetWin_1") - == target.SURFACE_BROWSER - ) - assert ( - target.resolve_surface(process_name="WXWork.exe", class_name="WeWorkWindow") - == target.SURFACE_WINDOWS - ) - assert get_driver(target.SURFACE_WINDOWS).driver_name == "uiautomation" - assert get_driver(target.SURFACE_BROWSER).driver_name == "playwright" - mobile = MobileDriver(surface=target.SURFACE_ANDROID) - try: - mobile.dump(info=None, selection_meta={}) - raise AssertionError("mobile dump should raise") - except RuntimeError as exc: - assert "SURFACE_UNSUPPORTED" in str(exc) - print("OK surface routing") - - -def _assert_user_prefs() -> None: - with tempfile.TemporaryDirectory(prefix="ui_prefs_") as tmp: - path = Path(tmp) / "user_prefs.json" - # monkeypatch prefs path - old = user_prefs._PREFS_PATH - try: - user_prefs._PREFS_PATH = path - user_prefs.save_prefs(mode="windows", start_url="https://example.com") - loaded = user_prefs.load_prefs() - assert loaded["mode"] == "windows" - assert loaded["start_url"] == "https://example.com" - user_prefs.save_prefs(mode="browser", start_url="") - loaded_empty = user_prefs.load_prefs() - assert loaded_empty["start_url"] == "" - user_prefs.save_prefs(mode="mobile", start_url="https://a.test") - loaded2 = user_prefs.load_prefs() - assert loaded2["mode"] == target.SURFACE_ANDROID - finally: - user_prefs._PREFS_PATH = old - print("OK user_prefs") - - -def _assert_skip_policy() -> None: - # blank + empty locatable → skip - assert ( - skip_policy.should_skip_empty_dump( - surface="browser", - url="about:blank", - locatable_count=0, - nodes_kept=1, - sparse=True, - ) - == "blank_or_newtab_empty" - ) - # blank URL but rich content (nav race like step05) → keep - assert ( - skip_policy.should_skip_empty_dump( - surface="browser", - url="about:blank", - locatable_count=74, - nodes_kept=97, - sparse=False, - ) - is None - ) - # real page → keep - assert ( - skip_policy.should_skip_empty_dump( - surface="browser", - url="https://www.douyin.com/jingxuan", - locatable_count=150, - nodes_kept=160, - sparse=False, - ) - is None - ) - # windows never skipped by this policy - assert ( - skip_policy.should_skip_empty_dump( - surface="windows", - url="", - locatable_count=0, - nodes_kept=1, - sparse=True, - ) - is None - ) - # same URL with content is NOT skipped (scroll-safe) - assert ( - skip_policy.should_skip_empty_dump( - surface="browser", - url="https://www.douyin.com/jingxuan", - locatable_count=20, - nodes_kept=30, - sparse=False, - ) - is None - ) - print("OK skip_policy") - - -def _assert_active_page_resolve() -> None: - """Unit-level: visible tab wins over sticky pages[0].""" - - class _FakePage: - def __init__(self, url: str, visible: bool, *, closed: bool = False) -> None: - self.url = url - self._visible = visible - self._closed = closed - - def is_closed(self) -> bool: - return self._closed - - def evaluate(self, _script: str) -> str: - return "visible" if self._visible else "hidden" - - def title(self) -> str: - return self.url - - class _FakeCtx: - def __init__(self, pages: list) -> None: - self.pages = pages - - search = _FakePage("https://www.douyin.com/search/foo", visible=False) - detail = _FakePage("https://www.douyin.com/video/123", visible=True) - ctx = _FakeCtx([search, detail]) - - page, meta = kit_browser.resolve_active_page(ctx, preferred=search) - assert page is detail, (page.url, meta) - assert meta["resolve"] == "visibility" - assert meta["page_count"] == 2 - assert meta["visible_count"] == 1 - - # No visible → preferred if still open - both_hidden = _FakeCtx( - [ - _FakePage("https://a.test/1", visible=False), - _FakePage("https://a.test/2", visible=False), - ] - ) - preferred = both_hidden.pages[0] - page2, meta2 = kit_browser.resolve_active_page(both_hidden, preferred=preferred) - assert page2 is preferred - assert meta2["resolve"] == "preferred_fallback" - - # Closed preferred → first open - closed_pref = _FakePage("https://gone", visible=False, closed=True) - open_only = _FakePage("https://alive", visible=False) - page3, meta3 = kit_browser.resolve_active_page( - _FakeCtx([closed_pref, open_only]), preferred=closed_pref - ) - assert page3 is open_only - assert meta3["resolve"] == "first_open" - print("OK active page resolve") - - -def _assert_page_dump_kit() -> None: - if not kit_browser.kit_page_dump_available(): - raise RuntimeError( - "jiangchang_skill_core.rpa.page_dump 不可用；" - "请 pip install -e jiangchang-platform-kit>=1.2.11" - ) - result = build_page_dump_result( - url="https://example.com", - title="Example", - a11y_snapshot={ - "role": "RootWebArea", - "name": "Example", - "children": [{"role": "button", "name": "OK", "children": []}], - }, - dom_nodes=[ - { - "tag": "button", - "role": "button", - "id": "", - "name_attr": "", - "testid": "", - "aria_label": "OK", - "placeholder": "", - "title": "", - "alt": "", - "href": "", - "type": "submit", - "text": "OK", - "disabled": False, - "rect": {}, - } - ], - ) - assert result["stats"]["locatable_count"] >= 1 - assert result["locatable"][0].get("primary_locator") - print("OK page_dump kit builder") - - -def _assert_label() -> None: - msg_page = [ - {"control_type": "TreeItemControl", "name": "groupbuddyname", "value": "某群"}, - {"control_type": "TreeItemControl", "name": "send_rich_edit", "value": "你好"}, - {"control_type": "TreeItemControl", "name": "con_op", "value": "消息"}, - ] - searching = [ - {"control_type": "TreeItemControl", "name": "groupbuddyname", "value": "徐书建"}, - {"control_type": "TreeControl", "name": "searchedit", "value": "陈德炼"}, - {"control_type": "TreeItemControl", "name": "send_rich_edit", "value": "陈德炼"}, - ] - selected = [ - { - "control_type": "TreeItemControl", - "name": "groupbuddyname", - "value": "陈德炼@微信", - }, - {"control_type": "TreeItemControl", "name": "send_rich_edit", "value": "您好"}, - ] - contacts = [ - {"control_type": "TreeItemControl", "name": "panel_title_label", "value": "RPA之家"}, - {"control_type": "TreeItemControl", "name": "main_chattext", "value": "新的客户"}, - ] - new_customer = [ - {"control_type": "TreeItemControl", "name": "panel_title_label", "value": "新的客户"}, - {"control_type": "TreeItemControl", "name": "ww_btn_text", "value": "添加"}, - ] - add_empty = [ - {"control_type": "TreeItemControl", "name": "title", "value": "添加客户"}, - {"control_type": "TreeControl", "name": "searchedit", "value": ""}, - {"control_type": "WindowControl", "name": "添加客户", "value": ""}, - ] - add_result = [ - {"control_type": "TreeItemControl", "name": "title", "value": "添加客户"}, - {"control_type": "TreeControl", "name": "searchedit", "value": "13316463772"}, - {"control_type": "TreeItemControl", "name": "main_chattext", "value": "Nancy"}, - {"control_type": "TreeItemControl", "name": "ww_btn_text", "value": "添加为"}, - {"control_type": "TreeControl", "name": "search_result", "value": ""}, - ] - - a = art.derive_step_label(window_title="企业微信", locatable=msg_page) - b = art.derive_step_label(window_title="企业微信", locatable=searching) - c = art.derive_step_label(window_title="企业微信", locatable=selected) - d = art.derive_step_label(window_title="企业微信", locatable=contacts) - e = art.derive_step_label(window_title="企业微信", locatable=new_customer) - f = art.derive_step_label(window_title="添加客户", locatable=add_empty) - g = art.derive_step_label(window_title="添加客户", locatable=add_result) - - assert "输入你好" in a or "某群" in a, a - assert "搜索陈德炼" in b, b - assert "陈德炼" in c and "输入您好" in c, c - assert "RPA之家" in d, d - assert "新的客户" in e and "添加" in e, e - assert f.startswith("添加客户"), f - assert "13316463772" in g or "搜索13316463772" in g, g - assert "会话列表" not in a and "会话列表" not in b and "会话列表" not in c - print(f"OK labels:\n {a}\n {b}\n {c}\n {d}\n {e}\n {f}\n {g}") - - -def main() -> int: - _assert_selection_api() - _assert_kit_wake_detect() - _assert_surface_routing() - _assert_user_prefs() - _assert_skip_policy() - _assert_active_page_resolve() - _assert_page_dump_kit() - _assert_label() - with tempfile.TemporaryDirectory(prefix="ui_flow_smoke_") as tmp: - root = Path(tmp) - flow_dir = art.create_flow(root, flow_name="demo-flow") - flow = art.load_flow(flow_dir) - - loc = [ - { - "control_type": "TreeItemControl", - "name": "groupbuddyname", - "value": "陈德炼@微信", - }, - {"control_type": "TreeItemControl", "name": "send_rich_edit", "value": "您好"}, - ] - label = art.resolve_step_label( - flow, window_title="企业微信", window_class="WeWorkWindow", locatable=loc - ) - step_dir, idx, name = art.add_step_dir( - flow_dir, - flow, - window_title="企业微信", - window_class="WeWorkWindow", - locatable=loc, - label=label, - ) - assert idx == 1 - assert "陈德炼" in name, name - art.write_json( - step_dir / "meta.json", - art.build_meta( - window={"title": "企业微信", "class_name": "WeWorkWindow", "pid": 1, "hwnd": 2}, - scene="demo", - surface="windows", - driver="uiautomation", - ), - ) - art.register_step( - flow_dir, - flow, - step_index=idx, - step_name=name, - window={"title": "企业微信", "class_name": "WeWorkWindow", "pid": 1, "hwnd": 2}, - stats={"findall_count": 10, "locatable_count": 5, "nodes_kept": 3, "uia_sparse": False}, - label=label, - surface="windows", - ) - # browser-shaped step registration - blow = art.load_flow(flow_dir) - art.register_step( - flow_dir, - blow, - step_index=2, - step_name="02_browser-demo", - window={"title": "精选", "class_name": "PlaywrightPage", "pid": 3, "hwnd": 4}, - stats={"findall_count": 20, "locatable_count": 12, "nodes_kept": 20, "uia_sparse": False}, - label="浏览器精选", - surface="browser", - url="https://www.douyin.com/jingxuan", - ) - flow = art.end_flow(flow_dir, art.load_flow(flow_dir)) - assert flow["status"] == "ended" - summary = (flow_dir / "SUMMARY.md").read_text(encoding="utf-8") - assert "surface: `browser`" in summary or "surface: `windows`" in summary - print(f"OK flow={flow_dir.name} step={name}") - print("OK ui_inspector flow smoke passed") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/ui_inspector/snapshot_service.py b/tools/ui_inspector/snapshot_service.py deleted file mode 100644 index 56348e4..0000000 --- a/tools/ui_inspector/snapshot_service.py +++ /dev/null @@ -1,400 +0,0 @@ -"""Snapshot service: wrap SurfaceDriver.dump into deduped UIState assets.""" - -from __future__ import annotations - -import time -from dataclasses import asdict, is_dataclass -from pathlib import Path -from typing import Any, Optional - -from models.observation import UIObservation -from models.serialize import to_dict, write_model_json -from models.state import UIState, compute_state_fingerprint -from redaction import ( - redact_locatable_rows, - sanitize_artifact, - screenshot_sensitivity_meta, -) - - -def _dump_to_parts(dump: Any) -> dict[str, Any]: - if dump is None: - return {} - if isinstance(dump, dict): - return dict(dump) - # DumpResult dataclass or duck-typed dump objects (tests / drivers). - if is_dataclass(dump) and not isinstance(dump, type) or hasattr(dump, "surface"): - return { - "surface": getattr(dump, "surface", "") or "", - "driver": getattr(dump, "driver", "") or "", - "window": dict(getattr(dump, "window", None) or {}), - "tree": getattr(dump, "tree", None) or {}, - "index": list(getattr(dump, "index", None) or []), - "locatable": list(getattr(dump, "locatable", None) or []), - "stats": dict(getattr(dump, "stats", None) or {}), - "selection": dict(getattr(dump, "selection", None) or {}), - "url": getattr(dump, "url", "") or "", - "page_title": getattr(dump, "page_title", "") or "", - "screenshot_bytes": getattr(dump, "screenshot_bytes", None), - "use_hwnd_screenshot": bool(getattr(dump, "use_hwnd_screenshot", False)), - "locatable_text": getattr(dump, "locatable_text", "") or "", - "tree_text": getattr(dump, "tree_text", "") or "", - } - return {} - - -class SnapshotService: - """ - Creates UIState records with fingerprint dedupe. - - Reuses SurfaceDriver.dump(); does not reimplement browser/desktop dump. - """ - - def __init__( - self, - flow_dir: Path, - *, - save_sensitive_screenshots: bool = False, - persist_evidence: bool = True, - ) -> None: - self.flow_dir = Path(flow_dir) - self.states_dir = self.flow_dir / "states" - self.states_dir.mkdir(parents=True, exist_ok=True) - self.observations_dir = self.flow_dir / "observations" - self.observations_dir.mkdir(parents=True, exist_ok=True) - self.save_sensitive_screenshots = save_sensitive_screenshots - self.persist_evidence = persist_evidence - self._by_fp: dict[str, UIState] = {} - self._order: list[str] = [] - self.latest: Optional[UIState] = None - self.latest_observation: Optional[UIObservation] = None - self._observation_order: list[str] = [] - - def load_existing(self) -> None: - """Index existing state JSON files (best-effort).""" - if not self.states_dir.exists(): - return - for path in sorted(self.states_dir.glob("*.json")): - try: - import json - - data = json.loads(path.read_text(encoding="utf-8")) - st = UIState( - id=data.get("id") or path.stem, - surface=data.get("surface") or "windows", - driver=data.get("driver") or "", - captured_at=data.get("captured_at") or "", - fingerprint=data.get("fingerprint") or "", - previous_state_id=data.get("previous_state_id"), - change_reason=data.get("change_reason") or "", - app=data.get("app") or {}, - window=data.get("window") or {}, - url=data.get("url") or "", - page_title=data.get("page_title") or "", - tree_path=data.get("tree_path") or "", - index_path=data.get("index_path") or "", - locatable_path=data.get("locatable_path") or "", - screenshot_path=data.get("screenshot_path") or "", - stats=data.get("stats") or {}, - selection=data.get("selection") or {}, - duplicate_of=data.get("duplicate_of"), - observation_count=int(data.get("observation_count") or 1), - may_contain_sensitive_data=bool( - data.get("may_contain_sensitive_data") - ), - ) - if st.fingerprint: - self._by_fp[st.fingerprint] = st - if st.id not in self._order: - self._order.append(st.id) - self.latest = st - except Exception: - continue - for path in sorted(self.observations_dir.glob("*/meta.json")): - try: - import json - - data = json.loads(path.read_text(encoding="utf-8")) - obs = UIObservation( - **{k: v for k, v in data.items() if k in UIObservation.__dataclass_fields__} - ) - self._observation_order.append(obs.id) - self.latest_observation = obs - except Exception: - continue - - def capture_from_dump( - self, - dump: Any, - *, - change_reason: str = "", - force_persist: bool = False, - previous_state_id: Optional[str] = None, - redact: bool = True, - ) -> UIState: - parts = _dump_to_parts(dump) - locatable = list(parts.get("locatable") or []) - sensitive = False - if redact: - locatable, sensitive = redact_locatable_rows(locatable) - - window = dict(parts.get("window") or {}) - url = parts.get("url") or "" - page_title = parts.get("page_title") or "" - surface = parts.get("surface") or "windows" - driver = parts.get("driver") or "" - fp = compute_state_fingerprint( - surface=surface, - driver=driver, - url=url, - page_title=page_title, - window=window, - locatable=locatable, - app={ - "title": window.get("title"), - "class_name": window.get("class_name"), - "process_name": window.get("process_name"), - }, - ) - - existing = self._by_fp.get(fp) - if existing is not None: - existing.bump_observation() - existing.duplicate_of = None - # Update meta path only — do not rewrite full tree evidence. - write_model_json(self.states_dir / f"{existing.id}.json", existing) - self.latest = existing - self._persist_observation( - existing, - parts, - locatable=locatable, - sensitive=sensitive, - change_reason=change_reason or "capture", - ) - return existing - - prev = previous_state_id - if prev is None and self.latest is not None: - prev = self.latest.id - - state = UIState( - surface=surface, - driver=driver, - fingerprint=fp, - previous_state_id=prev, - change_reason=change_reason or "capture", - app={ - "title": window.get("title"), - "class_name": window.get("class_name"), - "process_name": window.get("process_name"), - "pid": window.get("pid"), - "hwnd": window.get("hwnd"), - }, - window=window, - url=url, - page_title=page_title, - stats=dict(parts.get("stats") or {}), - selection=dict(parts.get("selection") or {}), - locatable=locatable, - may_contain_sensitive_data=sensitive, - ) - state.ensure_fingerprint() - - should_write_evidence = force_persist or bool(change_reason) or self.persist_evidence - if should_write_evidence: - self._persist_evidence(state, parts, sensitive=sensitive) - - write_model_json(self.states_dir / f"{state.id}.json", state) - self._by_fp[fp] = state - self._order.append(state.id) - self.latest = state - self._persist_observation( - state, - parts, - locatable=locatable, - sensitive=sensitive, - change_reason=change_reason or "capture", - ) - return state - - def _persist_observation( - self, - state: UIState, - parts: dict[str, Any], - *, - locatable: list[dict[str, Any]], - sensitive: bool, - change_reason: str, - ) -> UIObservation: - stats = dict(parts.get("stats") or {}) - selection = dict(parts.get("selection") or {}) - active_page = dict(stats.get("active_page") or {}) - browser_session = dict(stats.get("browser_session") or {}) - obs = UIObservation( - state_id=state.id, - previous_observation_id=( - self.latest_observation.id if self.latest_observation else None - ), - change_reason=change_reason, - surface=state.surface, - driver=state.driver, - page_id=str( - active_page.get("page_id") - or browser_session.get("page_id") - or selection.get("page_id") - or "" - ), - document_id=str(selection.get("document_id") or ""), - frame_id=str(selection.get("frame_id") or ""), - url=state.url, - page_title=state.page_title, - stats=stats, - selection=selection, - may_contain_sensitive_data=sensitive, - ) - evidence_dir = self.observations_dir / obs.id - evidence_dir.mkdir(parents=True, exist_ok=True) - import artifacts as art - - art.write_json( - evidence_dir / "tree.json", - sanitize_artifact(parts.get("tree") or {}), - ) - clean_index = sanitize_artifact(list(parts.get("index") or [])) - art.write_jsonl(evidence_dir / "index.jsonl", list(clean_index or [])) - art.write_json(evidence_dir / "locatable.json", list(locatable or [])) - obs.tree_path = f"observations/{obs.id}/tree.json" - obs.index_path = f"observations/{obs.id}/index.jsonl" - obs.locatable_path = f"observations/{obs.id}/locatable.json" - - sensitivity_meta = screenshot_sensitivity_meta( - may_contain_sensitive=sensitive, - save_sensitive_screenshots=self.save_sensitive_screenshots, - ) - art.write_json(evidence_dir / "sensitivity.json", sensitivity_meta) - if (not sensitive) or self.save_sensitive_screenshots: - png = parts.get("screenshot_bytes") - if png: - try: - (evidence_dir / "screen.png").write_bytes(png) - obs.screenshot_path = f"observations/{obs.id}/screen.png" - except Exception: - pass - elif parts.get("use_hwnd_screenshot"): - hwnd = int((parts.get("window") or {}).get("hwnd") or 0) - if hwnd and art.capture_window_png(hwnd, evidence_dir / "screen.png"): - obs.screenshot_path = f"observations/{obs.id}/screen.png" - - write_model_json(evidence_dir / "meta.json", obs) - self.latest_observation = obs - self._observation_order.append(obs.id) - return obs - - def _persist_evidence( - self, - state: UIState, - parts: dict[str, Any], - *, - sensitive: bool, - ) -> None: - evidence_dir = self.states_dir / state.id - evidence_dir.mkdir(parents=True, exist_ok=True) - import artifacts as art - - art.write_json( - evidence_dir / "tree.json", - sanitize_artifact(parts.get("tree") or {}), - ) - clean_index = sanitize_artifact(list(parts.get("index") or [])) - art.write_jsonl(evidence_dir / "index.jsonl", list(clean_index or [])) - art.write_json(evidence_dir / "locatable.json", list(state.locatable or [])) - state.tree_path = f"states/{state.id}/tree.json" - state.index_path = f"states/{state.id}/index.jsonl" - state.locatable_path = f"states/{state.id}/locatable.json" - - sens_meta = screenshot_sensitivity_meta( - may_contain_sensitive=sensitive, - save_sensitive_screenshots=self.save_sensitive_screenshots, - ) - art.write_json(evidence_dir / "sensitivity.json", sens_meta) - - save_png = (not sensitive) or self.save_sensitive_screenshots - if save_png: - png = parts.get("screenshot_bytes") - if png: - try: - (evidence_dir / "screen.png").write_bytes(png) - state.screenshot_path = f"states/{state.id}/screen.png" - except Exception: - pass - elif parts.get("use_hwnd_screenshot"): - hwnd = int((parts.get("window") or {}).get("hwnd") or 0) - if hwnd: - ok = art.capture_window_png(hwnd, evidence_dir / "screen.png") - if ok: - state.screenshot_path = f"states/{state.id}/screen.png" - elif sensitive: - state.may_contain_sensitive_data = True - - def wait_until_stable( - self, - dump_fn, - *, - timeout_sec: float = 3.0, - poll_sec: float = 0.25, - change_reason: str = "after_action", - ) -> UIState: - """ - Poll dump_fn until two consecutive fingerprints match or timeout. - dump_fn() -> DumpResult / dump-like object - """ - deadline = time.monotonic() + max(0.2, float(timeout_sec)) - last_fp: Optional[str] = None - last_dump = None - while True: - last_dump = dump_fn() - parts = _dump_to_parts(last_dump) - locatable, _ = redact_locatable_rows(list(parts.get("locatable") or [])) - fp = compute_state_fingerprint( - surface=parts.get("surface") or "windows", - driver=parts.get("driver") or "", - url=parts.get("url") or "", - page_title=parts.get("page_title") or "", - window=parts.get("window") or {}, - locatable=locatable, - ) - if last_fp is not None and fp == last_fp: - break - last_fp = fp - if time.monotonic() >= deadline: - break - time.sleep(max(0.05, float(poll_sec))) - return self.capture_from_dump( - last_dump, - change_reason=change_reason, - force_persist=True, - ) - - def state_ids(self) -> list[str]: - return list(self._order) - - def observation_ids(self) -> list[str]: - return list(self._observation_order) - - def get(self, state_id: str) -> Optional[UIState]: - for st in self._by_fp.values(): - if st.id == state_id: - return st - path = self.states_dir / f"{state_id}.json" - if path.exists(): - import json - from models.state import UIState as S - - data = json.loads(path.read_text(encoding="utf-8")) - return S(**{k: v for k, v in data.items() if k in S.__dataclass_fields__}) - return None - - def to_summary_dict(self, state: UIState) -> dict[str, Any]: - d = to_dict(state) - d["locatable"] = list(state.locatable or []) - return d diff --git a/tools/ui_inspector/state_diff.py b/tools/ui_inspector/state_diff.py deleted file mode 100644 index fa2627a..0000000 --- a/tools/ui_inspector/state_diff.py +++ /dev/null @@ -1,275 +0,0 @@ -"""State Diff + proposed postconditions.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Any, Optional - - -@dataclass -class StateChange: - kind: str - detail: dict[str, Any] = field(default_factory=dict) - confidence: float = 0.7 - - -@dataclass -class ProposedPostcondition: - description: str - kind: str - status: str = "proposed" # proposed | user_confirmed | runtime_verified - detail: dict[str, Any] = field(default_factory=dict) - - -def _row_key(r: dict[str, Any]) -> str: - return "|".join( - [ - str(r.get("control_type") or r.get("role") or ""), - str(r.get("automation_id") or ""), - str(r.get("name") or ""), - str(r.get("class_name") or ""), - ] - ) - - -def _row_value(r: dict[str, Any]) -> str: - return str(r.get("value") or r.get("text") or "") - - -def _index_rows(rows: Optional[list[dict[str, Any]]]) -> dict[str, dict[str, Any]]: - out: dict[str, dict[str, Any]] = {} - for r in rows or []: - if not isinstance(r, dict): - continue - k = _row_key(r) - if k.strip("|"): - out[k] = r - return out - - -def diff_states( - before: Optional[dict[str, Any]], - after: Optional[dict[str, Any]], -) -> list[StateChange]: - """ - Diff two state-like dicts (UIState.to_dict or lightweight summaries). - - Expected keys: url, page_title, window, locatable, stats, selection - """ - b = before or {} - a = after or {} - changes: list[StateChange] = [] - - b_url = str(b.get("url") or "") - a_url = str(a.get("url") or "") - if b_url != a_url: - changes.append( - StateChange( - kind="url_changed", - detail={"from": b_url, "to": a_url}, - ) - ) - - b_title = str(b.get("page_title") or (b.get("window") or {}).get("title") or "") - a_title = str(a.get("page_title") or (a.get("window") or {}).get("title") or "") - if b_title != a_title: - changes.append( - StateChange( - kind="title_changed", - detail={"from": b_title, "to": a_title}, - ) - ) - - b_cls = str((b.get("window") or {}).get("class_name") or "") - a_cls = str((a.get("window") or {}).get("class_name") or "") - b_hwnd = (b.get("window") or {}).get("hwnd") - a_hwnd = (a.get("window") or {}).get("hwnd") - if b_cls and a_cls and b_cls != a_cls: - changes.append( - StateChange( - kind="window_changed", - detail={"from_class": b_cls, "to_class": a_cls}, - ) - ) - if b_hwnd and not a_hwnd: - changes.append(StateChange(kind="window_disappeared", detail={"class_name": b_cls})) - if a_hwnd and not b_hwnd: - changes.append(StateChange(kind="window_appeared", detail={"class_name": a_cls})) - - # Popup heuristics by class name - for label, cls in (("before", b_cls), ("after", a_cls)): - del label - if "SearchResult" in a_cls and "SearchResult" not in b_cls: - changes.append( - StateChange(kind="popup_appeared", detail={"class_name": a_cls}) - ) - if "SearchResult" in b_cls and "SearchResult" not in a_cls: - changes.append( - StateChange(kind="popup_disappeared", detail={"class_name": b_cls}) - ) - - b_rows = _index_rows(b.get("locatable") or []) - a_rows = _index_rows(a.get("locatable") or []) - b_keys = set(b_rows) - a_keys = set(a_rows) - - for k in sorted(a_keys - b_keys)[:40]: - r = a_rows[k] - changes.append( - StateChange( - kind="element_appeared", - detail={ - "key": k, - "name": r.get("name"), - "value": _row_value(r), - "control_type": r.get("control_type") or r.get("role"), - "automation_id": r.get("automation_id"), - }, - ) - ) - for k in sorted(b_keys - a_keys)[:40]: - r = b_rows[k] - changes.append( - StateChange( - kind="element_disappeared", - detail={ - "key": k, - "name": r.get("name"), - "value": _row_value(r), - "control_type": r.get("control_type") or r.get("role"), - "automation_id": r.get("automation_id"), - }, - ) - ) - for k in sorted(b_keys & a_keys)[:80]: - bv = _row_value(b_rows[k]) - av = _row_value(a_rows[k]) - if bv != av: - changes.append( - StateChange( - kind="element_value_changed", - detail={ - "key": k, - "name": a_rows[k].get("name"), - "from": bv, - "to": av, - "automation_id": a_rows[k].get("automation_id"), - }, - ) - ) - - b_focus = (b.get("selection") or {}).get("focused") or (b.get("selection") or {}).get( - "focus" - ) - a_focus = (a.get("selection") or {}).get("focused") or (a.get("selection") or {}).get( - "focus" - ) - if b_focus != a_focus and (b_focus or a_focus): - changes.append( - StateChange( - kind="focus_changed", - detail={"from": b_focus, "to": a_focus}, - ) - ) - - b_loc = int((b.get("stats") or {}).get("locatable_count") or len(b.get("locatable") or [])) - a_loc = int((a.get("stats") or {}).get("locatable_count") or len(a.get("locatable") or [])) - if b_loc != a_loc: - changes.append( - StateChange( - kind="locatable_count_changed", - detail={"from": b_loc, "to": a_loc}, - ) - ) - - return changes - - -def propose_postconditions(changes: list[StateChange]) -> list[ProposedPostcondition]: - """Generate suggested postconditions (always status=proposed).""" - props: list[ProposedPostcondition] = [] - for ch in changes: - kind = ch.kind - d = ch.detail or {} - if kind == "url_changed": - props.append( - ProposedPostcondition( - description=f"URL became {d.get('to')!r}", - kind="url_equals", - detail=d, - ) - ) - elif kind == "popup_disappeared": - props.append( - ProposedPostcondition( - description=f"Window/popup {d.get('class_name')!r} disappeared", - kind="window_absent", - detail=d, - ) - ) - elif kind == "popup_appeared": - props.append( - ProposedPostcondition( - description=f"Window/popup {d.get('class_name')!r} appeared", - kind="window_present", - detail=d, - ) - ) - elif kind == "title_changed": - props.append( - ProposedPostcondition( - description=f"Title became {d.get('to')!r}", - kind="title_equals", - detail=d, - ) - ) - elif kind == "element_appeared": - name = d.get("name") or d.get("automation_id") or d.get("value") - props.append( - ProposedPostcondition( - description=f"Element {name!r} appeared", - kind="element_present", - detail=d, - ) - ) - elif kind == "element_disappeared": - name = d.get("name") or d.get("automation_id") or d.get("value") - props.append( - ProposedPostcondition( - description=f"Element {name!r} disappeared", - kind="element_absent", - detail=d, - ) - ) - elif kind == "element_value_changed": - props.append( - ProposedPostcondition( - description=( - f"Element {d.get('name') or d.get('automation_id')!r} " - f"value became {d.get('to')!r}" - ), - kind="element_value_equals", - detail=d, - ) - ) - return props - - -def changes_to_dicts(changes: list[StateChange]) -> list[dict[str, Any]]: - return [ - {"kind": c.kind, "detail": c.detail, "confidence": c.confidence} for c in changes - ] - - -def postconditions_to_dicts( - props: list[ProposedPostcondition], -) -> list[dict[str, Any]]: - return [ - { - "description": p.description, - "kind": p.kind, - "status": p.status, - "detail": p.detail, - } - for p in props - ] diff --git a/tools/ui_inspector/storage/__init__.py b/tools/ui_inspector/storage/__init__.py deleted file mode 100644 index ed1d34f..0000000 --- a/tools/ui_inspector/storage/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Capture persistence API.""" - -from core.pipeline.event_journal import EventJournal - -__all__ = ["EventJournal"] - diff --git a/tools/ui_inspector/storage/capture_reader.py b/tools/ui_inspector/storage/capture_reader.py deleted file mode 100644 index 14e4bc8..0000000 --- a/tools/ui_inspector/storage/capture_reader.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Stable capture reader used by diagnostics and downstream AI tooling.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - - -def read_json(path: Path) -> dict[str, Any]: - return json.loads(Path(path).read_text(encoding="utf-8")) - - -def read_action_flow(flow_dir: Path) -> dict[str, Any]: - return read_json(Path(flow_dir) / "action_flow.json") - - -def read_element_catalog(flow_dir: Path) -> dict[str, Any]: - return read_json(Path(flow_dir) / "element_catalog.json") - diff --git a/tools/ui_inspector/storage/capture_writer.py b/tools/ui_inspector/storage/capture_writer.py deleted file mode 100644 index a7715c6..0000000 --- a/tools/ui_inspector/storage/capture_writer.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Compatibility export for the current atomic artifact writer.""" - -from artifacts import create_flow, load_flow, save_flow, write_json - -__all__ = ["create_flow", "load_flow", "save_flow", "write_json"] - diff --git a/tools/ui_inspector/surfaces/__init__.py b/tools/ui_inspector/surfaces/__init__.py deleted file mode 100644 index cd3ed55..0000000 --- a/tools/ui_inspector/surfaces/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""Automation surface implementations.""" - diff --git a/tools/ui_inspector/surfaces/base.py b/tools/ui_inspector/surfaces/base.py deleted file mode 100644 index 788a49f..0000000 --- a/tools/ui_inspector/surfaces/base.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Surface contracts shared by browser, Windows, and future mobile drivers.""" - -from __future__ import annotations - -from typing import Any, Protocol - - -class SurfaceRecorder(Protocol): - status: str - def start(self) -> None: ... - def stop(self) -> None: ... - def pause(self) -> None: ... - def resume(self) -> None: ... - - -class SurfaceDumpDriver(Protocol): - surface: str - def dump(self, **kwargs: Any) -> Any: ... - diff --git a/tools/ui_inspector/surfaces/browser/__init__.py b/tools/ui_inspector/surfaces/browser/__init__.py deleted file mode 100644 index b9e1ade..0000000 --- a/tools/ui_inspector/surfaces/browser/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Browser surface public API.""" - -from .recorder import BrowserRecorder -from .dump_driver import BrowserPlaywrightDriver - -__all__ = ["BrowserRecorder", "BrowserPlaywrightDriver"] diff --git a/tools/ui_inspector/surfaces/browser/dump_driver.py b/tools/ui_inspector/surfaces/browser/dump_driver.py deleted file mode 100644 index 736038d..0000000 --- a/tools/ui_inspector/surfaces/browser/dump_driver.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Browser full-dump driver export.""" - -from drivers.browser_pw import BrowserPlaywrightDriver - -__all__ = ["BrowserPlaywrightDriver"] - diff --git a/tools/ui_inspector/surfaces/browser/locator_compiler.py b/tools/ui_inspector/surfaces/browser/locator_compiler.py deleted file mode 100644 index e3b2d79..0000000 --- a/tools/ui_inspector/surfaces/browser/locator_compiler.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Browser locator compiler export.""" - -from locators.browser import compile_browser_locators - -__all__ = ["compile_browser_locators"] - diff --git a/tools/ui_inspector/surfaces/browser/locator_validator.py b/tools/ui_inspector/surfaces/browser/locator_validator.py deleted file mode 100644 index a53395f..0000000 --- a/tools/ui_inspector/surfaces/browser/locator_validator.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Browser live locator validator export.""" - -from kit_browser import validate_element_on_page - -__all__ = ["validate_element_on_page"] - diff --git a/tools/ui_inspector/surfaces/browser/page_registry.py b/tools/ui_inspector/surfaces/browser/page_registry.py deleted file mode 100644 index 1a22a70..0000000 --- a/tools/ui_inspector/surfaces/browser/page_registry.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Stable browser page/frame identity facade.""" - -from kit_browser import page_id_for - -__all__ = ["page_id_for"] - diff --git a/tools/ui_inspector/surfaces/browser/recorder.py b/tools/ui_inspector/surfaces/browser/recorder.py deleted file mode 100644 index 266a3ce..0000000 --- a/tools/ui_inspector/surfaces/browser/recorder.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Browser recorder compatibility export during staged migration.""" - -from recorder.browser import BrowserRecorder - -__all__ = ["BrowserRecorder"] - diff --git a/tools/ui_inspector/surfaces/browser/runtime.py b/tools/ui_inspector/surfaces/browser/runtime.py deleted file mode 100644 index 2983be9..0000000 --- a/tools/ui_inspector/surfaces/browser/runtime.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Inspector-owned Playwright runtime facade.""" - -from kit_browser import ( - call_on_each_open_page, - call_on_page, - clear_event_bridge, - dump_current_page, - ensure_session, - install_event_bridge, - page_id_for, - screenshot_png, - session_info, - shutdown, - validate_element_on_page, -) - -__all__ = [ - "call_on_each_open_page", "call_on_page", "clear_event_bridge", - "dump_current_page", "ensure_session", "install_event_bridge", - "page_id_for", "screenshot_png", "session_info", "shutdown", - "validate_element_on_page", -] - diff --git a/tools/ui_inspector/surfaces/mobile/__init__.py b/tools/ui_inspector/surfaces/mobile/__init__.py deleted file mode 100644 index 170c18e..0000000 --- a/tools/ui_inspector/surfaces/mobile/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Future mobile surface public API.""" - -from drivers.mobile import MobileDriver - -__all__ = ["MobileDriver"] - diff --git a/tools/ui_inspector/surfaces/registry.py b/tools/ui_inspector/surfaces/registry.py deleted file mode 100644 index c05e1a4..0000000 --- a/tools/ui_inspector/surfaces/registry.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Resolve a dump driver through the canonical Surface namespace.""" - -from __future__ import annotations - -import target -from .browser.dump_driver import BrowserPlaywrightDriver -from .mobile import MobileDriver -from .windows.dump_driver import WindowsUiaDriver - - -def get_driver(surface: str): - if surface == target.SURFACE_BROWSER: - return BrowserPlaywrightDriver() - if surface == target.SURFACE_WINDOWS: - return WindowsUiaDriver() - if surface in {target.SURFACE_ANDROID, target.SURFACE_IOS}: - return MobileDriver(surface=surface) - raise RuntimeError(f"ERROR:SURFACE_UNSUPPORTED unknown surface={surface!r}") - - -__all__ = ["get_driver"] - diff --git a/tools/ui_inspector/surfaces/windows/__init__.py b/tools/ui_inspector/surfaces/windows/__init__.py deleted file mode 100644 index dafcdd2..0000000 --- a/tools/ui_inspector/surfaces/windows/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Windows surface public API.""" - -from .recorder import WindowsRecorder -from .dump_driver import WindowsUiaDriver - -__all__ = ["WindowsRecorder", "WindowsUiaDriver"] diff --git a/tools/ui_inspector/surfaces/windows/dump_driver.py b/tools/ui_inspector/surfaces/windows/dump_driver.py deleted file mode 100644 index 6284039..0000000 --- a/tools/ui_inspector/surfaces/windows/dump_driver.py +++ /dev/null @@ -1,4 +0,0 @@ -from drivers.windows_uia import WindowsUiaDriver - -__all__ = ["WindowsUiaDriver"] - diff --git a/tools/ui_inspector/surfaces/windows/locator_compiler.py b/tools/ui_inspector/surfaces/windows/locator_compiler.py deleted file mode 100644 index 4c99636..0000000 --- a/tools/ui_inspector/surfaces/windows/locator_compiler.py +++ /dev/null @@ -1,4 +0,0 @@ -from locators.windows import compile_windows_locators - -__all__ = ["compile_windows_locators"] - diff --git a/tools/ui_inspector/surfaces/windows/recorder.py b/tools/ui_inspector/surfaces/windows/recorder.py deleted file mode 100644 index 007b4cf..0000000 --- a/tools/ui_inspector/surfaces/windows/recorder.py +++ /dev/null @@ -1,4 +0,0 @@ -from recorder.windows import WindowsRecorder - -__all__ = ["WindowsRecorder"] - diff --git a/tools/ui_inspector/target.py b/tools/ui_inspector/target.py deleted file mode 100644 index 9eadda2..0000000 --- a/tools/ui_inspector/target.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Resolve capture surface from a Win32 window (windows | browser | mobile stub).""" - -from __future__ import annotations - -from typing import Any, Optional - -# Logical surfaces (align with RPA_ELEMENT_LOCATING.md) -SURFACE_WINDOWS = "windows" -SURFACE_BROWSER = "browser" -SURFACE_ANDROID = "android" -SURFACE_IOS = "ios" - -_BROWSER_PROCESS_NAMES = { - "chrome.exe", - "msedge.exe", - "chromium.exe", - "brave.exe", - "opera.exe", - "vivaldi.exe", -} - -_BROWSER_CLASS_HINTS = { - "Chrome_WidgetWin_1", - "Chrome_WidgetWin_0", - "Chrome_WidgetWin_2", -} - - -def normalize_process_name(name: str) -> str: - return (name or "").strip().lower() - - -def is_browser_window( - *, - process_name: str = "", - class_name: str = "", - title: str = "", -) -> bool: - proc = normalize_process_name(process_name) - if proc in _BROWSER_PROCESS_NAMES: - return True - cls = (class_name or "").strip() - if cls in _BROWSER_CLASS_HINTS: - return True - # Edge/Chrome sometimes expose empty process briefly; title heuristics last - t = title or "" - if " - Google Chrome" in t or " - Microsoft Edge" in t or t.endswith(" - Edge"): - return True - if t.endswith(" - Chrome"): - return True - return False - - -def resolve_surface( - *, - process_name: str = "", - class_name: str = "", - title: str = "", -) -> str: - """Return surface id for this top-level window.""" - if is_browser_window( - process_name=process_name, class_name=class_name, title=title - ): - return SURFACE_BROWSER - return SURFACE_WINDOWS - - -def surface_from_window_info(info: Any) -> str: - return resolve_surface( - process_name=getattr(info, "process_name", "") or "", - class_name=getattr(info, "class_name", "") or "", - title=getattr(info, "title", "") or "", - ) - - -def surface_label(surface: str) -> str: - return { - SURFACE_WINDOWS: "桌面 UIA", - SURFACE_BROWSER: "浏览器 Playwright", - SURFACE_ANDROID: "Android（未实现）", - SURFACE_IOS: "iOS（未实现）", - }.get(surface, surface) diff --git a/tools/ui_inspector/tests/__init__.py b/tools/ui_inspector/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tools/ui_inspector/tests/contract/__init__.py b/tools/ui_inspector/tests/contract/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/tools/ui_inspector/tests/contract/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tools/ui_inspector/tests/contract/test_capture_contract.py b/tools/ui_inspector/tests/contract/test_capture_contract.py deleted file mode 100644 index 8551b7c..0000000 --- a/tools/ui_inspector/tests/contract/test_capture_contract.py +++ /dev/null @@ -1,207 +0,0 @@ -from pathlib import Path - -from flow_session import FlowSession -from recorder.base import RawUserEvent -from diagnostics.capture_validator import validate_capture -from diagnostics.replay_preflight import build_replay_preflight - - -def test_session_always_creates_raw_event_journal(tmp_path: Path) -> None: - session = FlowSession(tmp_path, flow_name="contract", surface="browser", dump_fn=None) - session.manager.start() - session.manager.status = "recording" - event = RawUserEvent(kind="click", surface="browser", target={"role": "button", "name": "Go"}) - session.journal.append(event, channel="semantic") - assert (tmp_path / "journal" / "raw_events.jsonl").exists() - - -def test_validator_flags_oversized_click_target(tmp_path: Path) -> None: - import json - actions = tmp_path / "actions" - actions.mkdir() - (tmp_path / "elements").mkdir() - (tmp_path / "observations").mkdir() - (tmp_path / "journal").mkdir() - (tmp_path / "journal" / "raw_events.jsonl").write_text("{}\n", encoding="utf-8") - (actions / "a.json").write_text( - json.dumps({ - "id": "a", "sequence": 1, "action_type": "click", - "target_snapshot": {"rect": {"width": 1200, "height": 5000}}, - "evidence": {"eventual_observation_id": "obs"}, - }), - encoding="utf-8", - ) - result = validate_capture(tmp_path) - assert any(issue["code"] == "CLICK_TARGET_OVERSIZED" for issue in result["issues"]) - - -def test_validator_flags_missing_desktop_input_and_generic_targets(tmp_path: Path) -> None: - import json - - (tmp_path / "actions").mkdir() - (tmp_path / "elements").mkdir() - (tmp_path / "observations").mkdir() - (tmp_path / "journal").mkdir() - journal = { - "event": { - "kind": "typing_activity", "surface": "windows", - "target": {"name": "searchedit", "control_type": "TreeControl"}, - } - } - (tmp_path / "journal" / "raw_events.jsonl").write_text( - json.dumps(journal) + "\n", encoding="utf-8" - ) - rows = [ - { - "id": "a1", "sequence": 1, "surface": "windows", "action_type": "click", - "target_snapshot": { - "name": "search_result", "control_type": "TreeControl", - "bounds": {"width": 600, "height": 500}, - }, - "evidence": {"eventual_observation_id": "obs"}, - }, - { - "id": "a2", "sequence": 2, "surface": "windows", "action_type": "press_key", - "target_snapshot": { - "name": "WeCom", "control_type": "WindowControl", - "bounds": {"width": 1440, "height": 900}, - }, - "input": {"key": "Enter"}, - "evidence": {"eventual_observation_id": "obs"}, - }, - ] - for row in rows: - (tmp_path / "actions" / f"{row['id']}.json").write_text( - json.dumps(row), encoding="utf-8" - ) - codes = {issue["code"] for issue in validate_capture(tmp_path)["issues"]} - assert "CLICK_TARGET_GENERIC_CONTAINER" in codes - assert "KEY_TARGET_ROOT_WINDOW" in codes - assert "TEXT_INPUT_TRANSACTION_MISSING" in codes - replay = build_replay_preflight(tmp_path) - assert replay["ready"] is False - assert replay["quality_gate"]["passed"] is False - assert replay["summary"]["quality_blockers"] >= 1 - - -def test_validator_flags_missing_production_locator_and_raw_link(tmp_path: Path) -> None: - import json - - actions = tmp_path / "actions" - elements = tmp_path / "elements" - actions.mkdir() - elements.mkdir() - (tmp_path / "observations").mkdir() - (tmp_path / "journal").mkdir() - (tmp_path / "journal" / "raw_events.jsonl").write_text("{}\n", encoding="utf-8") - (elements / "el.json").write_text( - json.dumps({"id": "el", "primary_locator": None}), encoding="utf-8" - ) - (actions / "a.json").write_text( - json.dumps({ - "id": "a", "sequence": 1, "surface": "browser", - "action_type": "click", "target_element_id": "el", - "target_snapshot": {"role": "link", "rect": {"width": 20, "height": 20}}, - "raw_event": {}, "evidence": {"eventual_observation_id": "obs"}, - }), - encoding="utf-8", - ) - result = validate_capture(tmp_path) - codes = {issue["code"] for issue in result["issues"]} - assert "ACTION_PRODUCTION_LOCATOR_MISSING" in codes - assert "ACTION_RAW_EVENT_LINK_MISSING" in codes - - -def test_validator_flags_graphic_leaf_low_stability_and_split_fill(tmp_path: Path) -> None: - import json - - actions = tmp_path / "actions" - elements = tmp_path / "elements" - actions.mkdir() - elements.mkdir() - (tmp_path / "observations").mkdir() - (tmp_path / "journal").mkdir() - (tmp_path / "journal" / "raw_events.jsonl").write_text("{}\n", encoding="utf-8") - locator = { - "strategy": "css", "expression": "css:.hash", "actual_count": 1, - "unique": True, "production_eligible": True, "stability_score": 0.32, - "validation": "live", - } - (elements / "el.json").write_text( - json.dumps({"id": "el", "primary_locator": locator}), encoding="utf-8" - ) - rows = [ - { - "id": "a1", "sequence": 1, "surface": "browser", "action_type": "fill", - "target_element_id": "el", "target_snapshot": {"tag": "input"}, - "input": {"text": "a"}, - "raw_event": {"meta": {"event_id": "doc:1", "source_ts_ms": 1000}}, - "evidence": {"eventual_observation_id": "obs", "locator_unique": True}, - }, - { - "id": "a2", "sequence": 2, "surface": "browser", "action_type": "fill", - "target_element_id": "el", "target_snapshot": {"tag": "input"}, - "input": {"text": "ai"}, - "raw_event": {"meta": {"event_id": "doc:2", "source_ts_ms": 1100}}, - "evidence": {"eventual_observation_id": "obs", "locator_unique": True}, - }, - { - "id": "a3", "sequence": 3, "surface": "browser", "action_type": "click", - "target_element_id": "el", "target_snapshot": {"tag": "path"}, - "raw_event": {"meta": {"event_id": "doc:3"}}, - "evidence": {"eventual_observation_id": "obs", "locator_unique": True}, - }, - ] - for row in rows: - (actions / f"{row['id']}.json").write_text(json.dumps(row), encoding="utf-8") - codes = {issue["code"] for issue in validate_capture(tmp_path)["issues"]} - assert "DUPLICATE_FILL_TRANSACTION" in codes - assert "GRAPHIC_LEAF_TARGET" in codes - assert "LOW_STABILITY_LOCATOR" in codes - - -def test_replay_preflight_is_honest_about_event_time_validation(tmp_path: Path) -> None: - import json - - (tmp_path / "actions").mkdir() - (tmp_path / "elements").mkdir() - (tmp_path / "actions" / "a.json").write_text(json.dumps({ - "id": "a", "sequence": 1, "action_type": "click", - "target_element_id": "el", "raw_event_ids": ["doc:1"], - "raw_event": {"meta": {"page_id": "page", "document_id": "doc"}}, - }), encoding="utf-8") - (tmp_path / "elements" / "el.json").write_text(json.dumps({ - "id": "el", "logical_name": "go", "primary_locator": { - "strategy": "role", "expression": "role:button,Go", - "actual_count": 1, "unique": True, "production_eligible": True, - "stability_score": 0.95, "validation": "live", - }, - }), encoding="utf-8") - report = build_replay_preflight(tmp_path) - assert report["mode"] == "static_non_mutating" - assert report["summary"]["pass"] == 1 - assert report["actions"][0]["verification"] == "event_time_unique" - - -def test_replay_preflight_rejects_unique_root_window_click(tmp_path: Path) -> None: - import json - - (tmp_path / "actions").mkdir() - (tmp_path / "elements").mkdir() - (tmp_path / "actions" / "a.json").write_text(json.dumps({ - "id": "a", "sequence": 1, "surface": "windows", "action_type": "click", - "target_element_id": "root", "target_snapshot": { - "control_type": "WindowControl", "name": "WeCom", - "bounds": {"width": 1440, "height": 900}, - }, - }), encoding="utf-8") - (tmp_path / "elements" / "root.json").write_text(json.dumps({ - "id": "root", "logical_name": "WeCom", "primary_locator": { - "strategy": "window_name_type", "expression": "window:WeCom", - "actual_count": 1, "unique": True, "production_eligible": True, - "stability_score": 0.9, "validation": "snapshot_only", - }, - }), encoding="utf-8") - report = build_replay_preflight(tmp_path) - assert report["summary"]["fail"] == 1 - assert report["actions"][0]["reason"] == "oversized_or_root_window_click_target" diff --git a/tools/ui_inspector/tests/test_locators_catalog.py b/tools/ui_inspector/tests/test_locators_catalog.py deleted file mode 100644 index 136974e..0000000 --- a/tools/ui_inspector/tests/test_locators_catalog.py +++ /dev/null @@ -1,215 +0,0 @@ -"""Locator compiler / catalog / uniqueness tests.""" - -from __future__ import annotations - -import sys -import tempfile -import unittest -from pathlib import Path - -_HERE = Path(__file__).resolve().parents[1] -if str(_HERE) not in sys.path: - sys.path.insert(0, str(_HERE)) - -from locators.browser import compile_browser_locators -from locators.catalog import ElementCatalogService -from locators.validator import choose_production_primary, validate_locator_against_rows -from locators.windows import compile_windows_locators -from models.locator import LocatorCandidate - - -class TestBrowserLocators(unittest.TestCase): - def test_ordering_prefers_role_and_testid(self) -> None: - el = { - "role": "button", - "name": "Submit", - "test_id": "submit-btn", - "text": "Submit", - "id": "x1", - } - rows = [ - {"role": "button", "name": "Submit", "test_id": "submit-btn", "text": "Submit", "id": "x1"}, - {"role": "button", "name": "Cancel", "test_id": "cancel-btn", "text": "Cancel"}, - ] - cands = compile_browser_locators(el, rows=rows) - strategies = [c.strategy for c in cands] - self.assertIn("role", strategies) - self.assertIn("test_id", strategies) - primary = choose_production_primary(cands) - self.assertIsNotNone(primary) - assert primary is not None - self.assertFalse(primary.uses_coordinates) - - def test_contenteditable_ignores_dynamic_value_and_uses_role_css(self) -> None: - cands = compile_browser_locators( - { - "role": "combobox", - "name": "user typed comment", - "text": "user typed comment", - "value": "user typed comment", - "contenteditable": True, - "css": "#comment-input [contenteditable=true]", - } - ) - role = next(c for c in cands if c.strategy == "role") - css = next(c for c in cands if c.strategy == "css") - self.assertEqual(role.properties, {"role": "combobox"}) - self.assertNotIn("user typed comment", role.expression) - self.assertFalse(css.production_eligible) - - def test_live_unique_css_becomes_executable_fallback(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - service = ElementCatalogService(Path(tmp), flow_id="f") - el = service.upsert_from_snapshot( - { - "role": "path", - "tag": "path", - "css": "#comment-input .submit svg path", - "fingerprint": "path|submit", - }, - surface="browser", - frame={"url": "https://example.test/video"}, - live_counts={"css": 1}, - ) - self.assertIsNotNone(el.primary_locator) - assert el.primary_locator is not None - self.assertEqual(el.primary_locator.strategy, "css") - self.assertTrue(el.primary_locator.unique) - - def test_link_href_beats_ambiguous_card_css(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - service = ElementCatalogService(Path(tmp), flow_id="f") - el = service.upsert_from_snapshot( - { - "role": "link", - "tag": "a", - "name": "A result card with dynamic text", - "href": "/detail/42", - "href_count": 1, - "css": "#results > li.card > a.card-link", - "css_count": 24, - "fingerprint": "a|card", - }, - surface="browser", - frame={"url": "https://example.test/search"}, - live_counts={"href": 1, "css": 24}, - ) - self.assertIsNotNone(el.primary_locator) - assert el.primary_locator is not None - self.assertEqual(el.primary_locator.strategy, "href") - self.assertEqual(el.primary_locator.actual_count, 1) - self.assertTrue(el.primary_locator.production_eligible) - - def test_scoped_contenteditable_stable_css_can_be_primary(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - service = ElementCatalogService(Path(tmp), flow_id="f") - el = service.upsert_from_snapshot( - { - "role": "combobox", - "tag": "div", - "contenteditable": True, - "stable_css": "#comment-form div[contenteditable]", - "stable_css_count": 1, - "css": "#comment-form > div.hash-a > div.hash-b", - "css_count": 1, - "fingerprint": "div|editor", - }, - surface="browser", - frame={"url": "https://example.test/detail"}, - live_counts={"stable_css": 1, "css": 1}, - ) - self.assertIsNotNone(el.primary_locator) - assert el.primary_locator is not None - self.assertEqual(el.primary_locator.strategy, "stable_css") - self.assertGreater(el.primary_locator.stability_score, 0.7) - - -class TestWindowsLocators(unittest.TestCase): - def test_ordering_and_name_value_split(self) -> None: - el = { - "automation_id": "searchedit", - "name": "searchedit", - "value": "张三", - "control_type": "EditControl", - "parent_chain": [{"name": "search_layout"}], - } - rows = [ - { - "automation_id": "searchedit", - "name": "searchedit", - "value": "张三", - "control_type": "EditControl", - } - ] - cands = compile_windows_locators( - el, rows=rows, window_scope={"class_name": "WeWorkWindow", "title": "企业微信"} - ) - strategies = [c.strategy for c in cands] - self.assertTrue(strategies[0].startswith("window_automation_id")) - self.assertIn("window_name_value_type", strategies) - primary = choose_production_primary(cands) - self.assertIsNotNone(primary) - assert primary is not None - self.assertEqual(primary.actual_count, 1) - - def test_coordinates_never_primary(self) -> None: - el = { - "name": "x", - "control_type": "ButtonControl", - "bounds": {"left": 1, "top": 2, "right": 3, "bottom": 4}, - } - # Two matches → ambiguous for name+type; coords present - rows = [ - {"name": "x", "control_type": "ButtonControl"}, - {"name": "x", "control_type": "ButtonControl"}, - ] - cands = compile_windows_locators(el, rows=rows) - primary = choose_production_primary(cands) - self.assertIsNone(primary) - self.assertTrue(any(c.uses_coordinates for c in cands)) - for c in cands: - if c.uses_coordinates: - self.assertFalse(c.production_eligible) - - -class TestUniqueness(unittest.TestCase): - def test_ambiguous_not_primary(self) -> None: - c = LocatorCandidate( - strategy="name", - properties={"name": "item", "control_type": "ButtonControl"}, - stability_score=0.8, - ) - rows = [ - {"name": "item", "control_type": "ButtonControl"}, - {"name": "item", "control_type": "ButtonControl"}, - ] - validate_locator_against_rows(c, rows) - self.assertTrue(c.ambiguous) - self.assertFalse(c.production_eligible) - self.assertIsNone(choose_production_primary([c])) - - -class TestCatalogMerge(unittest.TestCase): - def test_merge_observations(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - svc = ElementCatalogService(Path(tmp), flow_id="f1") - el = { - "automation_id": "ok", - "name": "ok", - "control_type": "ButtonControl", - "value": "", - } - rows = [el] - a = svc.upsert_from_snapshot( - el, surface="windows", window_scope={"class_name": "X"}, rows=rows - ) - b = svc.upsert_from_snapshot( - el, surface="windows", window_scope={"class_name": "X"}, rows=rows - ) - self.assertEqual(a.id, b.id) - self.assertEqual(b.observation_count, 2) - self.assertTrue(svc.catalog_path.exists()) - - -if __name__ == "__main__": - unittest.main() diff --git a/tools/ui_inspector/tests/test_models_fingerprint_diff.py b/tools/ui_inspector/tests/test_models_fingerprint_diff.py deleted file mode 100644 index dbf33d0..0000000 --- a/tools/ui_inspector/tests/test_models_fingerprint_diff.py +++ /dev/null @@ -1,295 +0,0 @@ -"""Phase 2 unit tests: models, fingerprint, dedupe, state diff, redaction.""" - -from __future__ import annotations - -import sys -import tempfile -import unittest -from pathlib import Path - -_HERE = Path(__file__).resolve().parents[1] -if str(_HERE) not in sys.path: - sys.path.insert(0, str(_HERE)) - -from models.action import ActionFlow, ActionRecord -from models.element import ElementRef, element_fingerprint -from models.locator import LocatorCandidate -from models.serialize import dumps_json, to_dict -from models.state import UIState, compute_state_fingerprint -from core.policies.element_identity import element_identity_key -from redaction import redact_input_payload, redact_text -from snapshot_service import SnapshotService -from state_diff import diff_states, propose_postconditions - - -class TestSerialization(unittest.TestCase): - def test_state_serialize(self) -> None: - st = UIState( - surface="browser", - driver="playwright", - url="https://example.com", - page_title="Example", - locatable=[{"control_type": "button", "name": "Go", "value": ""}], - ) - st.ensure_fingerprint() - raw = dumps_json(st) - self.assertIn("openclaw.ui_inspector.state.v1", raw) - self.assertIn("https://example.com", raw) - self.assertNotIn("\\u", raw) # ensure_ascii=False for CJK; ASCII ok either way - d = to_dict(st) - self.assertEqual(d["schema"], "openclaw.ui_inspector.state.v1") - self.assertTrue(d["fingerprint"]) - - def test_action_serialize(self) -> None: - act = ActionRecord( - action_type="click", - surface="windows", - before_state_id="st_a", - after_state_id="st_b", - sequence=1, - ) - d = to_dict(act) - self.assertEqual(d["schema"], "openclaw.ui_inspector.action.v1") - self.assertEqual(d["capability"], "implemented") - flow = ActionFlow(flow_id="f1", flow_name="demo", actions=[act.id]) - self.assertEqual(to_dict(flow)["schema"], "openclaw.ui_inspector.action_flow.v1") - - def test_element_locator_serialize(self) -> None: - loc = LocatorCandidate( - strategy="automation_id", - properties={"automation_id": "ok", "control_type": "Button"}, - expression="AutomationId=ok", - stability_score=0.9, - ) - loc.mark_counts(1) - el = ElementRef( - logical_name="ok_button", - surface="windows", - automation_id="ok", - control_type="ButtonControl", - locator_candidates=[loc], - primary_locator=loc, - ) - el.ensure_fingerprint() - d = to_dict(el) - self.assertEqual(d["schema"], "openclaw.ui_inspector.element.v1") - self.assertTrue(d["primary_locator"]["production_eligible"]) - - -class TestFingerprint(unittest.TestCase): - def test_stable_fingerprint(self) -> None: - kwargs = dict( - surface="browser", - url="https://a.test/x", - page_title="T", - window={"title": "T", "class_name": "Chrome", "process_name": "chrome.exe"}, - locatable=[ - {"control_type": "textbox", "name": "q", "value": "hello", "automation_id": "q"} - ], - ) - a = compute_state_fingerprint(**kwargs) - b = compute_state_fingerprint(**kwargs) - self.assertEqual(a, b) - - def test_session_fields_ignored(self) -> None: - base_loc = [{"control_type": "button", "name": "A", "value": "", "automation_id": "a"}] - fp1 = compute_state_fingerprint( - surface="windows", - window={"title": "App", "class_name": "X", "pid": 111, "hwnd": 222}, - locatable=base_loc, - ) - fp2 = compute_state_fingerprint( - surface="windows", - window={"title": "App", "class_name": "X", "pid": 999, "hwnd": 888}, - locatable=base_loc, - ) - self.assertEqual(fp1, fp2) - - def test_url_changes_fingerprint(self) -> None: - loc = [{"control_type": "button", "name": "A", "value": "", "automation_id": "a"}] - a = compute_state_fingerprint(surface="browser", url="https://a.test/1", locatable=loc) - b = compute_state_fingerprint(surface="browser", url="https://a.test/2", locatable=loc) - self.assertNotEqual(a, b) - - def test_anonymous_dom_elements_do_not_merge_across_paths(self) -> None: - common = dict(surface="browser", role="span", name="", value="") - a = element_fingerprint(**common, dom_fingerprint="span|#video-tab") - b = element_fingerprint(**common, dom_fingerprint="span|#comment-input") - self.assertNotEqual(a, b) - - def test_contenteditable_value_is_not_identity(self) -> None: - common = dict( - surface="browser", - role="combobox", - contenteditable=True, - dom_fingerprint="div|#comment-input [contenteditable]", - ) - self.assertEqual( - element_fingerprint(**common, name="first", value="first"), - element_fingerprint(**common, name="second", value="second"), - ) - - def test_link_href_is_identity_when_visible_text_changes(self) -> None: - common = { - "role": "link", - "tag": "a", - "href": "/detail/42", - "css": "#results > li.card > a", - } - self.assertEqual( - element_identity_key( - {**common, "name": "Result with 10 likes"}, - surface="browser", - page_url="https://example.test/search?page=1", - ), - element_identity_key( - {**common, "name": "Result with 11 likes"}, - surface="browser", - page_url="https://example.test/search?page=2", - ), - ) - - def test_value_change_detected(self) -> None: - a = compute_state_fingerprint( - surface="windows", - locatable=[{"control_type": "Edit", "name": "box", "value": "1", "automation_id": "e"}], - ) - b = compute_state_fingerprint( - surface="windows", - locatable=[{"control_type": "Edit", "name": "box", "value": "2", "automation_id": "e"}], - ) - self.assertNotEqual(a, b) - - -class TestDedupe(unittest.TestCase): - def test_same_state_deduped(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - svc = SnapshotService(Path(tmp), persist_evidence=False) - - class Dump: - surface = "browser" - driver = "playwright" - window = {"title": "T", "class_name": "P"} - tree = {} - index = [] - locatable = [ - {"control_type": "button", "name": "Go", "value": "", "automation_id": "go"} - ] - stats = {"locatable_count": 1, "nodes_kept": 1} - selection = {} - url = "https://example.com" - page_title = "T" - screenshot_bytes = None - use_hwnd_screenshot = False - - s1 = svc.capture_from_dump(Dump(), change_reason="entry") - s2 = svc.capture_from_dump(Dump(), change_reason="heartbeat") - self.assertEqual(s1.id, s2.id) - self.assertEqual(s2.observation_count, 2) - self.assertEqual(len(svc.state_ids()), 1) - - -class TestStateDiff(unittest.TestCase): - def test_diff_and_postconditions(self) -> None: - before = { - "url": "https://a.test/1", - "page_title": "One", - "window": {"class_name": "WeWorkWindow", "title": "企业微信", "hwnd": 1}, - "locatable": [ - { - "control_type": "Edit", - "name": "searchedit", - "value": "q", - "automation_id": "searchedit", - } - ], - "stats": {"locatable_count": 1}, - } - after = { - "url": "https://a.test/1", - "page_title": "Chat", - "window": {"class_name": "WeWorkWindow", "title": "企业微信", "hwnd": 1}, - "locatable": [ - { - "control_type": "Edit", - "name": "send_rich_edit", - "value": "", - "automation_id": "send_rich_edit", - } - ], - "stats": {"locatable_count": 1}, - } - changes = diff_states(before, after) - kinds = {c.kind for c in changes} - self.assertIn("title_changed", kinds) - self.assertIn("element_appeared", kinds) - self.assertIn("element_disappeared", kinds) - props = propose_postconditions(changes) - self.assertTrue(all(p.status == "proposed" for p in props)) - self.assertTrue(any("appeared" in p.description.lower() or "出现" in p.description or "appeared" in p.description for p in props) or props) - - -class TestRedaction(unittest.TestCase): - def test_password_redacted(self) -> None: - out, sens = redact_input_payload( - {"text": "secret123", "name": "password"}, - field_name="password", - input_type="password", - ) - self.assertTrue(sens) - self.assertEqual(out["text"], "[REDACTED]") - - def test_phone_redacted(self) -> None: - self.assertIn("[REDACTED]", redact_text("call 13800138000 please")) - - def test_raw_tree_and_index_are_sanitized_before_write(self) -> None: - secret = "Bearer_abcdefghijklmnopqrstuvwxyz123456" - - class SensitiveDump: - surface = "browser" - driver = "playwright" - window = {"title": "Login"} - tree = { - "role": "textbox", - "name": "password", - "input_type": "password", - "value": secret, - } - index = [dict(tree)] - locatable = [dict(tree)] - stats = {} - selection = {} - url = "https://example.com/login" - page_title = "Login" - screenshot_bytes = None - use_hwnd_screenshot = False - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - svc = SnapshotService(root) - svc.capture_from_dump( - SensitiveDump(), change_reason="sensitive", force_persist=True - ) - persisted = "\n".join( - p.read_text(encoding="utf-8", errors="ignore") - for p in root.rglob("*") - if p.is_file() - ) - self.assertNotIn(secret, persisted) - - -class TestActionBeforeAfter(unittest.TestCase): - def test_click_links_states(self) -> None: - act = ActionRecord( - action_type="click", - before_state_id="st_before", - after_state_id="st_after", - target_snapshot={"name": "btn"}, - ) - self.assertEqual(act.before_state_id, "st_before") - self.assertEqual(act.after_state_id, "st_after") - self.assertEqual(act.capability, "implemented") - - -if __name__ == "__main__": - unittest.main() diff --git a/tools/ui_inspector/tests/test_recorder_adapters.py b/tools/ui_inspector/tests/test_recorder_adapters.py deleted file mode 100644 index 23fe6c3..0000000 --- a/tools/ui_inspector/tests/test_recorder_adapters.py +++ /dev/null @@ -1,1513 +0,0 @@ -"""Recorder / aggregator / adapter / heartbeat tests.""" - -from __future__ import annotations - -import sys -import tempfile -import threading -import time -import unittest -from pathlib import Path -from unittest.mock import patch - -_HERE = Path(__file__).resolve().parents[1] -if str(_HERE) not in sys.path: - sys.path.insert(0, str(_HERE)) - -from adapters.wecom import wecom_adapter -from flow_session import FlowSession -from evidence.coordinator import EvidenceCoordinator -import kit_browser -from kit_wake import ( - _cache_key, - _caches, - auto_ensure_if_wecom, - note_provider_warm, - resolve_wake_profile, -) -from models.action import ActionRecord -from recorder.base import RawUserEvent, RecorderState -from recorder.browser import BrowserRecorder -from recorder.input_aggregator import InputAggregator -from recorder.manager import ActionRecordingManager -from recorder.windows import ( - WindowsRecorder, - _WM_LBUTTONDOWN, - _WM_LBUTTONUP, -) -from redaction import redact_element_fields, sanitize_artifact -from snapshot_service import SnapshotService - - -class Dump: - def __init__(self, url="https://example.com", value="x", title="T"): - self.surface = "browser" - self.driver = "playwright" - self.window = {"title": title, "class_name": "P"} - self.tree = {} - self.index = [] - self.locatable = [ - { - "control_type": "textbox", - "name": "q", - "value": value, - "automation_id": "q", - } - ] - self.stats = {"locatable_count": 1, "nodes_kept": 1} - self.selection = {} - self.url = url - self.page_title = title - self.screenshot_bytes = None - self.use_hwnd_screenshot = False - - -class TestInputAggregator(unittest.TestCase): - def test_aggregates_chars(self) -> None: - agg = InputAggregator(idle_sec=0.5) - out = [] - out += agg.on_key(key="a", text="a", target={"name": "q", "fingerprint": "1"}) - out += agg.on_key(key="b", text="ab", target={"name": "q", "fingerprint": "1"}) - self.assertEqual(out, []) - flushed = agg.flush() - self.assertEqual(len(flushed), 1) - self.assertEqual(flushed[0].action_type, "fill") - self.assertEqual(flushed[0].text, "ab") - - def test_enter_is_press_key(self) -> None: - agg = InputAggregator() - items = agg.on_key(key="Enter", target={"name": "q"}) - self.assertEqual(len(items), 1) - self.assertEqual(items[0].action_type, "press_key") - self.assertEqual(items[0].key, "Enter") - - def test_sensitive_fill(self) -> None: - agg = InputAggregator() - agg.on_key( - key="x", - text="secret", - target={"name": "password", "fingerprint": "p"}, - input_type="password", - field_name="password", - ) - items = agg.flush() - self.assertTrue(items[0].sensitive) - self.assertEqual(items[0].input_payload.get("text"), "[REDACTED]") - - def test_empty_value_change_records_clear_as_fill(self) -> None: - agg = InputAggregator() - agg.on_value_change( - value="", - target={"name": "comment", "fingerprint": "editor"}, - capture_context={"page_id": "page_detail"}, - ) - items = agg.flush() - self.assertEqual(len(items), 1) - self.assertEqual(items[0].action_type, "fill") - self.assertEqual(items[0].input_payload.get("text"), "") - self.assertEqual(items[0].capture_context.get("page_id"), "page_detail") - - def test_focus_class_change_stays_one_fill_transaction(self) -> None: - agg = InputAggregator(idle_sec=8.0) - context = { - "surface": "browser", - "url": "https://example.test/search", - "frame_name": "", - } - first = { - "role": "textbox", "tag": "input", "placeholder": "Search", - "class_name": "field", "fingerprint": "input|field", - "stable_css": '#header input[placeholder="Search"]', - } - focused = { - **first, - "class_name": "field focused", - "fingerprint": "input|field.focused", - } - out = agg.on_value_change( - value="a", target=first, capture_context=context, now=1000.0, - raw_event={"meta": {"event_id": "doc:9"}}, - ) - out += agg.on_value_change( - value="ai", target=focused, capture_context=context, now=1000.136, - raw_event={"meta": {"event_id": "doc:10"}}, - ) - self.assertEqual(out, []) - flushed = agg.flush(now=1000.2) - self.assertEqual(len(flushed), 1) - self.assertEqual(flushed[0].input_payload["text"], "ai") - self.assertEqual(flushed[0].raw_event_ids, ["doc:9", "doc:10"]) - - -class TestWindowsRecorder(unittest.TestCase): - def test_machine_element_name_is_not_redacted_as_a_secret(self) -> None: - cleaned, sensitive = redact_element_fields( - {"name": "OperationAndSendMessageRegion", "control_type": "PaneControl"} - ) - self.assertEqual(cleaned["name"], "OperationAndSendMessageRegion") - self.assertFalse(sensitive) - - def test_shadow_windows_are_decorative_not_business_overlays(self) -> None: - self.assertTrue(WindowsRecorder._is_decorative_window_class("PerryShadowWnd")) - self.assertTrue(WindowsRecorder._is_decorative_window_class("AcmeDropShadow")) - self.assertFalse(WindowsRecorder._is_decorative_window_class("SearchResultWindow2")) - - def test_persisted_atlas_prefers_exact_editor_over_layout(self) -> None: - recorder = WindowsRecorder() - bounds = { - "left": 336, "top": 660, "right": 1422, "bottom": 798, - "width": 1086, "height": 138, - } - recorder.learn_elements( - [ - {"name": "send_rich_edit_layout", "control_type": "TreeControl", "rect": bounds}, - {"name": "send_rich_edit", "control_type": "TreeControl", "rect": bounds}, - ], - window={"class_name": "WeWorkWindow", "hwnd": 10, "pid": 20}, - ) - hit = recorder._atlas_hit("WeWorkWindow", [(439, 673)]) - self.assertEqual(hit.get("name"), "send_rich_edit") - self.assertEqual(hit.get("value"), "") - - def test_typing_activity_survives_temporarily_unresolved_target(self) -> None: - seen = [] - recorder = WindowsRecorder(on_event=seen.append) - recorder.status = RecorderState.RECORDING - recorder._last_root_hwnd = 10 - recorder._last_uia_point = (100, 200) - recorder._emit_typing_activity({}, {"source_time_ms": 1}) - self.assertEqual([event.kind for event in seen], ["typing_activity"]) - self.assertTrue(seen[0].meta.get("unresolved_target")) - self.assertEqual(seen[0].meta.get("root_hwnd"), 10) - - def test_input_method_candidate_window_is_not_a_business_target(self) -> None: - self.assertTrue( - WindowsRecorder._is_transient_input_target( - { - "control_type": "PaneControl", - "class_name": "SoPY_Comp", - "window_scope": {"class_name": "SoPY_Comp"}, - } - ) - ) - self.assertFalse( - WindowsRecorder._is_transient_input_target( - {"name": "send_rich_edit", "control_type": "TreeControl"} - ) - ) - - def test_backspace_in_editor_is_folded_into_value_transaction(self) -> None: - seen = [] - recorder = WindowsRecorder(on_event=seen.append) - recorder.status = RecorderState.RECORDING - editor = { - "name": "send_rich_edit", "control_type": "TreeControl", - "window_scope": {"hwnd": 10}, - } - recorder._publish_focus_snapshot(editor) - recorder._on_key_down({"vk": 0x08, "source_time_ms": 1}) - self.assertEqual([event.kind for event in seen], ["typing_activity"]) - self.assertNotIn("Backspace", [event.key for event in seen]) - - def test_ime_enter_is_not_emitted_as_business_key(self) -> None: - seen = [] - recorder = WindowsRecorder(on_event=seen.append) - recorder.status = RecorderState.RECORDING - editor = { - "name": "send_rich_edit", "control_type": "TreeControl", - "window_scope": {"hwnd": 10, "pid": 20, "class_name": "WeWorkWindow"}, - } - recorder._last_text_entry_target = dict(editor) - - recorder._ime_composing = True - recorder._publish_focus_snapshot(editor) - recorder._on_key_down({"vk": 0x0D, "source_time_ms": 1}) - self.assertEqual([event.kind for event in seen], ["typing_activity"]) - self.assertNotIn("Enter", [event.key for event in seen]) - - def test_key_consumer_never_calls_uia(self) -> None: - seen = [] - recorder = WindowsRecorder(on_event=seen.append) - recorder.status = RecorderState.RECORDING - recorder._publish_focus_snapshot( - {"name": "send_rich_edit", "control_type": "TreeControl", "window_scope": {"hwnd": 10}} - ) - with patch.object( - recorder, "_focused_control_snapshot", side_effect=AssertionError("UIA on input thread") - ), patch.object( - recorder, "_poll_focus_value", side_effect=AssertionError("UIA on input thread") - ): - for _ in range(100): - recorder._on_key_down({"vk": 0x41, "source_time_ms": 1}) - self.assertEqual([event.kind for event in seen], ["typing_activity"]) - - def test_slow_uia_sampler_does_not_block_burst_input_consumer(self) -> None: - recorder = WindowsRecorder() - recorder.status = RecorderState.RECORDING - recorder._stop.clear() - recorder._uia_finished.clear() - slow_started = threading.Event() - - def slow_snapshot(): - slow_started.set() - time.sleep(0.45) - return {} - - with patch.object(recorder, "_focused_control_snapshot", side_effect=slow_snapshot): - consumer = threading.Thread(target=recorder._capture_loop) - sampler = threading.Thread(target=recorder._uia_sampler_loop) - consumer.start() - sampler.start() - recorder._uia_request.set() - self.assertTrue(slow_started.wait(0.2)) - for index in range(120): - recorder._enqueue_input({"kind": "key", "vk": 0x41, "source_time_ms": index}) - deadline = time.monotonic() + 0.3 - while recorder.health()["resolved_events"] < 120 and time.monotonic() < deadline: - time.sleep(0.005) - self.assertEqual(recorder.health()["resolved_events"], 120) - self.assertEqual(recorder._input_queue.qsize(), 0) - recorder._stop.set() - recorder._uia_request.set() - sampler.join(1.5) - consumer.join(1.5) - self.assertFalse(sampler.is_alive()) - self.assertFalse(consumer.is_alive()) - - def test_stop_drains_every_accepted_input_packet(self) -> None: - recorder = WindowsRecorder() - recorder.status = RecorderState.RECORDING - recorder._stop.clear() - recorder._uia_finished.clear() - with patch.object(recorder, "_focused_control_snapshot", return_value={}): - recorder._capture_thread = threading.Thread(target=recorder._capture_loop) - recorder._uia_thread = threading.Thread(target=recorder._uia_sampler_loop) - recorder._capture_thread.start() - recorder._uia_thread.start() - for index in range(300): - recorder._enqueue_input({"kind": "key", "vk": 0x41, "source_time_ms": index}) - recorder.stop() - health = recorder.health() - self.assertEqual(health["resolved_events"], 300) - self.assertEqual(health["undrained_events"], 0) - self.assertTrue(health["shutdown_complete"]) - - def test_value_is_flushed_before_enter_boundary(self) -> None: - seen = [] - recorder = WindowsRecorder(on_event=seen.append) - recorder.status = RecorderState.RECORDING - editor = { - "name": "send_rich_edit", "control_type": "TreeControl", "value": "hello", - "window_scope": {"hwnd": 10}, - } - recorder._last_focus_fp = recorder._focus_fingerprint(editor) - recorder._last_focus_value = "" - recorder._typing_active = True - recorder._publish_focus_snapshot(editor) - recorder._on_key_down({"vk": 0x0D, "source_time_ms": 2}) - self.assertEqual([event.kind for event in seen], ["value_change", "key"]) - self.assertEqual(seen[0].text, "hello") - - def test_focused_control_reprobes_mixed_dpi_point_after_provider_warms(self) -> None: - recorder = WindowsRecorder() - recorder._last_raw_point = (784, 1358) - recorder._last_uia_point = (782, 1357) - recorder._last_root_hwnd = 10 - root = { - "name": "WeCom", "control_type": "WindowControl", - "bounds": {"width": 2880, "height": 1800}, - "window_scope": {"hwnd": 10, "pid": 20, "class_name": "WeWorkWindow"}, - } - editor = { - "name": "send_rich_edit", "control_type": "TreeControl", - "value": "hello", "bounds": {"left": 336, "top": 679, "right": 1060, "bottom": 817, "width": 724, "height": 138}, - "window_scope": {"hwnd": 10, "pid": 20, "class_name": "WeWorkWindow"}, - } - fake_auto = type( - "FakeAuto", - (), - { - "GetFocusedControl": staticmethod(lambda: object()), - "ControlFromPoint": staticmethod(lambda *_args: object()), - }, - ) - with patch.dict(sys.modules, {"uiautomation": fake_auto}), patch.object( - recorder, "_control_snapshot", return_value=root - ), patch.object( - recorder, - "_probe_mixed_uia_coordinates", - return_value=(editor, 392, 679, {"coordinate_selection": "window_dpi_absolute"}), - ): - focused = recorder._focused_control_snapshot() - self.assertEqual(focused.get("name"), "send_rich_edit") - - def test_transient_focus_falls_back_to_last_real_editor(self) -> None: - recorder = WindowsRecorder() - editor = { - "name": "searchedit", "control_type": "TreeControl", - "window_scope": {"hwnd": 10, "pid": 20, "class_name": "WeWorkWindow"}, - } - transient = { - "class_name": "SoPY_Comp", "control_type": "PaneControl", - "bounds": {"width": 400, "height": 100}, - "window_scope": {"hwnd": 99, "pid": 20, "class_name": "SoPY_Comp"}, - } - recorder._last_text_entry_target = dict(editor) - fake_auto = type( - "FakeAuto", (), {"GetFocusedControl": staticmethod(lambda: object())} - ) - with patch.dict(sys.modules, {"uiautomation": fake_auto}), patch.object( - recorder, "_control_snapshot", return_value=transient - ): - focused = recorder._focused_control_snapshot() - self.assertEqual(focused.get("name"), "searchedit") - self.assertTrue(recorder._ime_composing) - - def test_focus_value_change_emits_committed_text(self) -> None: - seen = [] - recorder = WindowsRecorder(on_event=seen.append) - recorder.status = RecorderState.RECORDING - recorder._last_focus_fp = "|send_rich_edit|TreeControl" - recorder._last_focus_value = "" - editor = { - "name": "send_rich_edit", "control_type": "TreeControl", - "value": "您好", "window_scope": {"hwnd": 10}, - } - with patch.object(recorder, "_focused_control_snapshot", return_value=editor): - recorder._poll_focus_value() - self.assertEqual(len(seen), 1) - self.assertEqual(seen[0].kind, "value_change") - self.assertEqual(seen[0].text, "您好") - - def test_typing_activity_is_coalesced_and_contains_no_key_text(self) -> None: - seen = [] - recorder = WindowsRecorder(on_event=seen.append) - recorder.status = RecorderState.RECORDING - editor = { - "name": "searchedit", "control_type": "TreeControl", - "window_scope": {"hwnd": 10}, - } - recorder._emit_typing_activity(editor, {"source_time_ms": 1}) - recorder._emit_typing_activity(editor, {"source_time_ms": 2}) - self.assertEqual(len(seen), 1) - self.assertEqual(seen[0].kind, "typing_activity") - self.assertEqual(seen[0].text, "") - self.assertFalse(seen[0].meta["content_recorded"]) - - def test_wecom_root_hit_probes_window_dpi_coordinate_candidate(self) -> None: - recorder = WindowsRecorder() - root = { - "name": "WeCom", - "control_type": "WindowControl", - "class_name": "WeWorkWindow", - "bounds": {"width": 2880, "height": 1800}, - "window_scope": { - "hwnd": 10, "pid": 20, "class_name": "WeWorkWindow", - }, - } - search = { - "name": "searchedit", - "control_type": "TreeControl", - "bounds": {"width": 139, "height": 22}, - "window_scope": { - "hwnd": 10, "pid": 20, "class_name": "WeWorkWindow", - }, - } - with patch( - "recorder.windows.ctypes.windll.user32.GetDpiForWindow", - return_value=192, - ), patch.object(recorder, "_element_from_point", return_value=(search, "")): - target, x, y, meta = recorder._probe_mixed_uia_coordinates( - 10, 282, 76, current_point=(280, 72), current_target=root - ) - self.assertEqual((x, y), (141, 38)) - self.assertEqual(target.get("name"), "searchedit") - self.assertEqual(meta.get("coordinate_selection"), "window_dpi_absolute") - - def test_physical_hook_point_prefers_actual_uia_root_coordinate_space(self) -> None: - class Rect: - left, top, right, bottom = 0, 0, 1440, 900 - - class Root: - BoundingRectangle = Rect() - - fake_auto = type("FakeAuto", (), {"ControlFromHandle": staticmethod(lambda _hwnd: Root())}) - - def physical_rect(_hwnd, _attr, ptr, _size): - rect = ptr._obj - rect.left, rect.top, rect.right, rect.bottom = 0, 0, 2880, 1800 - return 0 - - with patch.dict(sys.modules, {"uiautomation": fake_auto}), patch( - "recorder.windows.ctypes.windll.dwmapi.DwmGetWindowAttribute", - side_effect=physical_rect, - ): - x, y, meta = WindowsRecorder._uia_point_for_window( - 123, 884, 1396, coordinate_space="physical_screen" - ) - self.assertEqual((x, y), (442, 698)) - self.assertEqual(meta["coordinate_conversion"], "dwm_physical_to_uia_root") - self.assertEqual(meta["dpi_scale"], {"x": 2.0, "y": 2.0}) - - def test_physical_hook_point_is_mapped_to_dpi_virtualized_uia_space(self) -> None: - def logical_rect(_hwnd, ptr): - rect = ptr._obj - rect.left, rect.top, rect.right, rect.bottom = -2, -2, 1442, 854 - return 1 - - def physical_rect(_hwnd, _attr, ptr, _size): - rect = ptr._obj - rect.left, rect.top, rect.right, rect.bottom = 0, 0, 2880, 1708 - return 0 - - with patch( - "recorder.windows.ctypes.windll.user32.GetWindowRect", - side_effect=logical_rect, - ), patch( - "recorder.windows.ctypes.windll.dwmapi.DwmGetWindowAttribute", - side_effect=physical_rect, - ): - x, y, meta = WindowsRecorder._uia_point_for_window( - 123, 363, 90, coordinate_space="physical_screen" - ) - self.assertTrue(175 <= x <= 185) - self.assertTrue(40 <= y <= 47) - self.assertAlmostEqual(meta["dpi_scale"]["x"], 2.0, delta=0.02) - - def test_mouse_packets_preserve_hook_coordinates_and_injected_flag(self) -> None: - seen = [] - recorder = WindowsRecorder(on_event=seen.append) - recorder.status = RecorderState.RECORDING - with patch.object( - recorder, - "_element_from_point", - return_value=({"name": "Send", "control_type": "ButtonControl"}, ""), - ) as element_from_point, patch.object(recorder, "exclude_hwnds", return_value=set()), patch( - "recorder.windows.ctypes.windll.user32.WindowFromPoint", return_value=1 - ), patch( - "recorder.windows.ctypes.windll.user32.GetAncestor", return_value=1 - ): - recorder._health_set(backend="low_level_hooks") - recorder._dispatch_input_packet( - { - "kind": "mouse", - "message": _WM_LBUTTONDOWN, - "x": 321, - "y": 654, - "injected": True, - "source_time_ms": 1234, - } - ) - recorder._dispatch_input_packet( - {"kind": "mouse", "message": _WM_LBUTTONUP} - ) - self.assertEqual(len(seen), 1) - self.assertEqual((seen[0].x, seen[0].y), (321, 654)) - self.assertTrue(seen[0].meta["injected"]) - self.assertEqual(seen[0].meta["input_backend"], "low_level_hooks") - element_from_point.assert_not_called() - - def test_health_exposes_backend_and_queue_counters(self) -> None: - recorder = WindowsRecorder() - recorder._health_set(backend="low_level_hooks", mouse_hook=True) - recorder._enqueue_input({"kind": "mouse", "message": _WM_LBUTTONUP}) - health = recorder.health() - self.assertEqual(health["backend"], "low_level_hooks") - self.assertTrue(health["mouse_hook"]) - self.assertEqual(health["raw_input_events"], 1) - self.assertEqual(health["mouse_events"], 1) - - def test_all_inspector_process_windows_are_excluded(self) -> None: - seen = [] - recorder = WindowsRecorder(on_event=seen.append) - recorder.status = RecorderState.RECORDING - - def set_current_pid(_hwnd, pid_ptr): - import os - pid_ptr._obj.value = os.getpid() - return 1 - - with patch( - "recorder.windows.ctypes.windll.user32.WindowFromPoint", return_value=10 - ), patch( - "recorder.windows.ctypes.windll.user32.GetAncestor", return_value=20 - ), patch( - "recorder.windows.ctypes.windll.user32.GetWindowThreadProcessId", - side_effect=set_current_pid, - ), patch.object(recorder, "_element_from_point") as element_from_point: - recorder._on_mouse_down_at(100, 200) - self.assertEqual(seen, []) - element_from_point.assert_not_called() - - -class TestActionManager(unittest.TestCase): - def test_enter_inherits_the_fill_target_instead_of_root_window(self) -> None: - editor = { - "name": "send_rich_edit", "control_type": "TreeControl", - "bounds": {"width": 500, "height": 100}, - "window_scope": {"hwnd": 10, "pid": 20, "class_name": "WeWorkWindow"}, - } - root = { - "name": "WeCom", "control_type": "WindowControl", - "bounds": {"width": 1440, "height": 900}, - "window_scope": {"hwnd": 10, "pid": 20, "class_name": "WeWorkWindow"}, - } - with tempfile.TemporaryDirectory() as tmp: - mgr = ActionRecordingManager(Path(tmp), surface="windows", dump_fn=None) - mgr.start() - mgr.handle_raw_event( - RawUserEvent(kind="value_change", surface="windows", text="您好", target=editor) - ) - key = mgr.handle_raw_event( - RawUserEvent(kind="key", surface="windows", key="Enter", target=root) - ) - actions = sorted(mgr._actions_by_id.values(), key=lambda action: action.sequence) - self.assertEqual([action.action_type for action in actions], ["fill", "press_key"]) - self.assertEqual(key.target_snapshot.get("name"), "send_rich_edit") - - def test_unique_result_row_is_preferred_over_generic_search_container(self) -> None: - chosen = ActionRecordingManager._unique_windows_row_candidate( - [ - { - "name": "search_result", "control_type": "TreeControl", - "rect": {"left": 0, "top": 0, "right": 500, "bottom": 500, "width": 500, "height": 500}, - }, - { - "depth": 8, "name": "user_item", "control_type": "TreeControl", - "value": "", "rect": {"left": 0, "top": 50, "right": 500, "bottom": 110, "width": 500, "height": 60}, - }, - { - "depth": 9, "name": "search_name", "control_type": "TreeItemControl", - "value": "陈德炼", "rect": {"left": 60, "top": 60, "right": 180, "bottom": 90, "width": 120, "height": 30}, - }, - ] - ) - self.assertIsNotNone(chosen) - self.assertEqual(chosen.get("name"), "user_item") - self.assertEqual(chosen.get("value"), "陈德炼") - self.assertEqual(chosen.get("semantic_name"), "陈德炼") - - def test_unique_row_prefers_named_value_and_strips_rich_text_markup(self) -> None: - chosen = ActionRecordingManager._unique_windows_row_candidate( - [ - { - "depth": 8, "name": "user_item", "control_type": "TreeControl", - "rect": {"left": 0, "top": 50, "right": 500, "bottom": 110, "width": 500, "height": 60}, - }, - { - "depth": 9, "name": "search_avatar_label", "value": "avatar", - "control_type": "TreeItemControl", - }, - { - "depth": 9, "name": "search_name", - "value": "Alice@WeCom", - "control_type": "TreeItemControl", - }, - ] - ) - self.assertIsNotNone(chosen) - self.assertEqual(chosen.get("semantic_name"), "Alice@WeCom") - self.assertEqual(chosen.get("value"), "Alice@WeCom") - - def test_named_editor_beats_smaller_generic_list_item_at_same_point(self) -> None: - chosen = ActionRecordingManager._windows_hit_candidate( - [ - { - "name": "listitem_content", "control_type": "ListItemControl", - "rect": {"left": 100, "top": 500, "right": 300, "bottom": 560, "width": 200, "height": 60}, - }, - { - "name": "send_rich_edit", "control_type": "TreeControl", - "rect": {"left": 100, "top": 500, "right": 900, "bottom": 700, "width": 800, "height": 200}, - }, - ], - 150, - 530, - ) - self.assertEqual(chosen.get("name"), "send_rich_edit") - - def test_coordinate_promotion_cleans_search_result_rich_text(self) -> None: - class PopupDump(Dump): - def __init__(self): - super().__init__() - self.surface = "windows" - self.window = { - "hwnd": 30, "pid": 20, "class_name": "SearchResultWindow2" - } - self.locatable = [ - { - "name": "search_name", "control_type": "TreeItemControl", - "value": "Alice@WeCom", - "rect": {"left": 0, "top": 0, "right": 300, "bottom": 60, "width": 300, "height": 60}, - } - ] - - broad = { - "name": "SearchResultWindow2", "control_type": "PaneControl", - "bounds": {"left": 0, "top": 0, "right": 600, "bottom": 600, "width": 600, "height": 600}, - "window_scope": {"hwnd": 30, "pid": 20, "class_name": "SearchResultWindow2"}, - } - with tempfile.TemporaryDirectory() as tmp: - mgr = ActionRecordingManager(Path(tmp), surface="windows", dump_fn=None, defer_action_evidence=True) - mgr.start(entry_dump=PopupDump()) - action = mgr.handle_raw_event( - RawUserEvent(kind="click", surface="windows", x=20, y=20, target=broad) - ) - assert action is not None - mgr.attach_eventual_evidence( - [action.id], dump=PopupDump(), - requested_monotonic=__import__("time").monotonic(), - ) - self.assertEqual(action.target_snapshot.get("value"), "Alice@WeCom") - - def test_machine_identifier_is_not_redacted_as_token(self) -> None: - cleaned = sanitize_artifact( - {"name": "conversation_list_container", "value": ""} - ) - self.assertEqual(cleaned.get("name"), "conversation_list_container") - - def test_action_observer_receives_capture_and_evidence_phases(self) -> None: - phases = [] - with tempfile.TemporaryDirectory() as tmp: - mgr = ActionRecordingManager( - Path(tmp), surface="browser", dump_fn=None, - defer_action_evidence=True, - on_action_update=lambda action, phase: phases.append((action.id, phase)), - ) - mgr.start(entry_dump=Dump(value="entry")) - action = mgr.handle_raw_event( - RawUserEvent( - kind="click", surface="browser", - target={"role": "button", "name": "Go", "css_count": 1}, - ) - ) - assert action is not None - mgr.attach_eventual_evidence( - [action.id], dump=Dump(value="after"), - requested_monotonic=__import__("time").monotonic(), - ) - self.assertEqual([phase for _action_id, phase in phases], ["captured", "evidence"]) - - def test_windows_hit_candidate_rejects_one_pixel_separator(self) -> None: - chosen = ActionRecordingManager._windows_hit_candidate( - [ - { - "control_type": "TreeItemControl", - "name": "conv_list_sep_line", - "rect": {"left": 0, "top": 10, "right": 300, "bottom": 11, "width": 300, "height": 1}, - }, - { - "control_type": "TreeItemControl", - "name": "groupbuddyname", - "rect": {"left": 0, "top": 0, "right": 300, "bottom": 60, "width": 300, "height": 60}, - }, - ], - 90, - 10, - ) - self.assertIsNotNone(chosen) - self.assertEqual(chosen.get("name"), "groupbuddyname") - - def test_eventual_dump_never_promotes_action_from_another_window(self) -> None: - class WindowsDump(Dump): - def __init__(self): - super().__init__() - self.surface = "windows" - self.window = { - "hwnd": 10, "pid": 20, - "title": "WeCom", "class_name": "WeWorkWindow", - } - self.locatable = [ - { - "control_type": "TreeControl", "name": "searchedit", - "rect": {"left": 0, "top": 0, "right": 300, "bottom": 80, "width": 300, "height": 80}, - } - ] - - with tempfile.TemporaryDirectory() as tmp: - mgr = ActionRecordingManager( - Path(tmp), surface="windows", dump_fn=None, - defer_action_evidence=True, - ) - mgr.start(entry_dump=WindowsDump()) - action = mgr.handle_raw_event( - RawUserEvent( - kind="click", surface="windows", x=20, y=20, - target={ - "control_type": "GroupControl", "class_name": "QWidget", - "bounds": {"left": 0, "top": 0, "right": 2880, "bottom": 1800, "width": 2880, "height": 1800}, - "window_scope": {"hwnd": 99, "pid": 30, "class_name": "Qt51514QWindowIcon"}, - }, - meta={"uia_coordinates": {"x": 20, "y": 20}}, - ) - ) - assert action is not None - mgr.attach_eventual_evidence( - [action.id], dump=WindowsDump(), - requested_monotonic=__import__("time").monotonic(), - ) - self.assertEqual(action.target_snapshot.get("class_name"), "QWidget") - self.assertNotIn("target_promotion", action.evidence) - - def test_eventual_windows_tree_promotes_root_click_to_smallest_target(self) -> None: - class WindowsDump(Dump): - def __init__(self): - super().__init__(value="after") - self.surface = "windows" - self.driver = "uiautomation" - self.window = { - "hwnd": 10, "pid": 20, - "title": "WeCom", "class_name": "WeWorkWindow", - } - self.locatable = [ - { - "control_type": "WindowControl", "name": "WeCom", - "class_name": "WeWorkWindow", - "bounds": {"left": 0, "top": 0, "right": 1440, "bottom": 900, "width": 1440, "height": 900}, - }, - { - "control_type": "TreeControl", "name": "searchedit", - "bounds": {"left": 110, "top": 25, "right": 249, "bottom": 47, "width": 139, "height": 22}, - }, - ] - - with tempfile.TemporaryDirectory() as tmp: - mgr = ActionRecordingManager( - Path(tmp), surface="windows", dump_fn=None, - defer_action_evidence=True, - ) - mgr.start(entry_dump=WindowsDump()) - action = mgr.handle_raw_event( - RawUserEvent( - kind="click", surface="windows", x=181, y=45, - target={ - "control_type": "WindowControl", "name": "WeCom", - "class_name": "WeWorkWindow", - "bounds": {"left": 0, "top": 0, "right": 1440, "bottom": 900, "width": 1440, "height": 900}, - }, - meta={"uia_coordinates": {"x": 181, "y": 45}}, - ) - ) - assert action is not None - mgr.attach_eventual_evidence( - [action.id], dump=WindowsDump(), requested_monotonic=__import__("time").monotonic() - ) - self.assertEqual(action.target_snapshot.get("name"), "searchedit") - self.assertEqual(action.target_snapshot.get("control_type"), "TreeControl") - self.assertEqual(action.evidence.get("target_promotion"), "eventual_tree_coordinate_hit") - self.assertTrue(action.target_element_id) - - def test_eventual_popup_tree_promotes_generic_container_to_unique_result_row(self) -> None: - class PopupDump(Dump): - def __init__(self): - super().__init__(value="after") - self.surface = "windows" - self.driver = "uiautomation" - self.window = { - "hwnd": 30, "pid": 20, - "title": "Search", "class_name": "SearchResultWindow2", - } - self.locatable = [ - { - "control_type": "TreeControl", "name": "search_result", - "rect": {"left": 0, "top": 0, "right": 1000, "bottom": 600, "width": 1000, "height": 600}, - }, - { - "control_type": "TreeControl", "name": "user_item", "value": "Alice", - "rect": {"left": 0, "top": 50, "right": 1000, "bottom": 110, "width": 1000, "height": 60}, - }, - ] - - with tempfile.TemporaryDirectory() as tmp: - mgr = ActionRecordingManager( - Path(tmp), surface="windows", dump_fn=None, - defer_action_evidence=True, - ) - mgr.start(entry_dump=PopupDump()) - action = mgr.handle_raw_event( - RawUserEvent( - kind="click", surface="windows", x=400, y=400, - target={ - "control_type": "TreeControl", "name": "search_result", - "bounds": {"left": 0, "top": 0, "right": 1000, "bottom": 600, "width": 1000, "height": 600}, - "window_scope": {"hwnd": 30, "pid": 20, "class_name": "SearchResultWindow2"}, - }, - meta={"uia_coordinates": {"x": 400, "y": 400}}, - ) - ) - assert action is not None - mgr.attach_eventual_evidence( - [action.id], dump=PopupDump(), - requested_monotonic=__import__("time").monotonic(), - ) - self.assertEqual(action.target_snapshot.get("name"), "user_item") - self.assertEqual(action.target_snapshot.get("value"), "Alice") - self.assertEqual(action.evidence.get("target_promotion"), "eventual_tree_unique_row") - - def test_main_window_never_uses_popup_unique_row_fallback(self) -> None: - class MainDump(Dump): - def __init__(self): - super().__init__(value="after") - self.surface = "windows" - self.driver = "uiautomation" - self.window = { - "hwnd": 10, "pid": 20, "title": "WeCom", - "class_name": "WeWorkWindow", - } - self.locatable = [ - { - "control_type": "ListItemControl", "name": "listitem_content", - "rect": {"left": 0, "top": 50, "right": 300, "bottom": 110, "width": 300, "height": 60}, - } - ] - - broad = { - "control_type": "WindowControl", "name": "WeCom", - "bounds": {"left": 0, "top": 0, "right": 1400, "bottom": 900, "width": 1400, "height": 900}, - "window_scope": {"hwnd": 10, "pid": 20, "class_name": "WeWorkWindow"}, - } - with tempfile.TemporaryDirectory() as tmp: - mgr = ActionRecordingManager( - Path(tmp), surface="windows", dump_fn=None, - defer_action_evidence=True, - ) - mgr.start(entry_dump=MainDump()) - action = mgr.handle_raw_event( - RawUserEvent( - kind="click", surface="windows", x=800, y=800, - target=broad, meta={"uia_coordinates": {"x": 800, "y": 800}}, - ) - ) - assert action is not None - mgr.attach_eventual_evidence( - [action.id], dump=MainDump(), - requested_monotonic=__import__("time").monotonic(), - ) - self.assertEqual(action.target_snapshot.get("name"), "WeCom") - self.assertNotEqual( - action.evidence.get("target_promotion"), "eventual_tree_unique_row" - ) - - def test_late_enter_evidence_uses_prior_editor_action_as_context(self) -> None: - class WindowsDump(Dump): - def __init__(self): - super().__init__(value="after") - self.surface = "windows" - self.driver = "uiautomation" - self.window = {"hwnd": 10, "pid": 20, "title": "WeCom", "class_name": "WeWorkWindow"} - self.locatable = [ - { - "control_type": "TreeControl", "name": "send_rich_edit", - "rect": {"left": 100, "top": 500, "right": 900, "bottom": 700, "width": 800, "height": 200}, - } - ] - - editor = { - "control_type": "TreeControl", "name": "send_rich_edit", - "bounds": {"left": 100, "top": 500, "right": 900, "bottom": 700, "width": 800, "height": 200}, - "window_scope": {"hwnd": 10, "pid": 20, "class_name": "WeWorkWindow"}, - } - root = { - "control_type": "WindowControl", "name": "WeCom", - "bounds": {"width": 1440, "height": 900}, - "window_scope": {"hwnd": 10, "pid": 20, "class_name": "WeWorkWindow"}, - } - with tempfile.TemporaryDirectory() as tmp: - mgr = ActionRecordingManager( - Path(tmp), surface="windows", dump_fn=None, - defer_action_evidence=True, - ) - mgr.start(entry_dump=WindowsDump()) - mgr.handle_raw_event(RawUserEvent(kind="click", surface="windows", target=editor)) - enter = mgr.handle_raw_event(RawUserEvent(kind="key", surface="windows", key="Enter", target=root)) - assert enter is not None - mgr.attach_eventual_evidence( - [enter.id], dump=WindowsDump(), - requested_monotonic=__import__("time").monotonic(), - ) - self.assertEqual(enter.target_snapshot.get("name"), "send_rich_edit") - - def test_late_dynamic_message_preview_cannot_rewrite_old_root_click(self) -> None: - class WindowsDump(Dump): - def __init__(self): - super().__init__(value="after") - self.surface = "windows" - self.driver = "uiautomation" - self.window = {"hwnd": 10, "pid": 20, "title": "WeCom", "class_name": "WeWorkWindow"} - self.locatable = [ - { - "control_type": "TreeItemControl", - "name": "groupbuddydescription", - "value": "a much later message preview", - "rect": {"left": 0, "top": 0, "right": 500, "bottom": 60, "width": 500, "height": 60}, - } - ] - - root = { - "control_type": "WindowControl", "name": "WeCom", - "bounds": {"width": 1440, "height": 900}, - "window_scope": {"hwnd": 10, "pid": 20, "class_name": "WeWorkWindow"}, - } - with tempfile.TemporaryDirectory() as tmp: - mgr = ActionRecordingManager( - Path(tmp), surface="windows", dump_fn=None, - defer_action_evidence=True, - ) - mgr.start(entry_dump=WindowsDump()) - action = mgr.handle_raw_event( - RawUserEvent( - kind="click", surface="windows", x=20, y=20, - target=root, meta={"uia_coordinates": {"x": 20, "y": 20}}, - ) - ) - assert action is not None - mgr.attach_eventual_evidence( - [action.id], dump=WindowsDump(), - requested_monotonic=__import__("time").monotonic() - 10.0, - ) - self.assertEqual(action.target_snapshot.get("control_type"), "WindowControl") - self.assertEqual(action.evidence.get("target_promotion_rejected"), "late_dynamic_candidate") - - def test_windows_deferred_evidence_keeps_action_hot_path_fast(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - mgr = ActionRecordingManager( - Path(tmp), surface="windows", dump_fn=None, - defer_action_evidence=True, - ) - mgr.start(entry_dump=Dump(value="entry")) - - def slow_dump(**_kwargs): - import time - time.sleep(0.2) - return Dump(value="eventual") - - mgr.dump_fn = slow_dump - started = __import__("time").monotonic() - action = mgr.handle_raw_event( - RawUserEvent( - kind="click", surface="windows", - target={"control_type": "ButtonControl", "name": "Send"}, - ) - ) - self.assertLess(__import__("time").monotonic() - started, 0.1) - self.assertIsNotNone(action) - - def test_windows_session_buffers_event_emitted_during_startup(self) -> None: - class FakeWindowsRecorder: - def __init__(self, *, on_event, exclude_hwnds): - self.on_event = on_event - self.status = RecorderState.IDLE - - def start(self): - self.status = RecorderState.RECORDING - self.on_event( - RawUserEvent( - kind="click", surface="windows", - target={"control_type": "ButtonControl", "name": "Early"}, - ) - ) - - def stop(self): - self.status = RecorderState.STOPPED - - with tempfile.TemporaryDirectory() as tmp, patch( - "surfaces.windows.WindowsRecorder", FakeWindowsRecorder - ): - session = FlowSession( - Path(tmp), flow_name="startup", surface="windows", dump_fn=None - ) - session.start() - session._event_queue.join() - self.assertEqual(len(session.manager.action_flow.actions), 1) - session.stop() - - def test_deferred_evidence_does_not_block_action_hot_path(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - mgr = ActionRecordingManager( - root, surface="browser", dump_fn=None, defer_action_evidence=True - ) - - def slow_dump(**_kwargs): - import time - time.sleep(0.2) - return Dump(value="eventual") - - evidence = EvidenceCoordinator(mgr, slow_dump, coalesce_sec=0.03) - mgr.on_action = evidence.schedule - mgr.start(entry_dump=Dump(value="entry")) - evidence.start() - import time - started = time.monotonic() - action = mgr.handle_raw_event( - RawUserEvent( - kind="click", surface="browser", - target={"role": "button", "name": "Go", "css_count": 1}, - meta={"page_id": "page_1"}, - ) - ) - elapsed = time.monotonic() - started - self.assertLess(elapsed, 0.1) - self.assertIsNotNone(action) - assert action is not None - self.assertIsNone(action.after_state_id) - evidence.stop(timeout=3.0) - self.assertIn("eventual_observation_id", action.evidence) - - def test_weak_comment_icon_gets_submit_semantic_hint(self) -> None: - class Before: - locatable = [ - {"role": "button", "name": "提交"}, - {"role": "button", "name": "关注"}, - ] - - target = ActionRecordingManager._enrich_weak_browser_target( - { - "role": "path", - "tag": "path", - "css": "#comment-input-container .commentInput-right-ct svg path", - }, - Before(), - ) - self.assertEqual(target.get("semantic_role"), "button") - self.assertEqual(target.get("semantic_name"), "提交") - - def test_press_key_keeps_raw_event_and_target_element(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - mgr = ActionRecordingManager( - Path(tmp), surface="browser", dump_fn=None, - defer_action_evidence=True, - ) - mgr.start() - action = mgr.handle_raw_event( - RawUserEvent( - kind="key", - surface="browser", - key="Enter", - url="https://example.test/search", - frame={"url": "https://example.test/search", "name": ""}, - target={ - "role": "textbox", - "tag": "input", - "placeholder": "Search", - "stable_css": 'input[placeholder="Search"]', - "stable_css_count": 1, - "fingerprint": "input|search", - }, - meta={ - "event_id": "doc:17", - "document_id": "doc", - "page_id": "page_1", - "sequence": 17, - }, - ) - ) - self.assertIsNotNone(action) - assert action is not None - self.assertTrue(action.target_element_id) - self.assertEqual(action.raw_event["meta"]["event_id"], "doc:17") - self.assertEqual(action.raw_event["meta"]["page_id"], "page_1") - self.assertEqual(action.evidence["locator_match_count"], 1) - self.assertTrue(action.evidence["locator_unique"]) - - def test_action_capture_uses_one_dump_not_stability_poll(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - calls = [] - - def dump_fn(): - calls.append(1) - return Dump(value="after") - - mgr = ActionRecordingManager( - Path(tmp), surface="windows", dump_fn=dump_fn - ) - mgr.start(entry_dump=Dump(value="before")) - mgr.handle_raw_event( - RawUserEvent( - kind="click", surface="windows", - target={"control_type": "ButtonControl", "name": "Go"}, - ) - ) - self.assertEqual(len(calls), 1) - - def test_click_before_after_and_no_heartbeat_dup(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - dumps = [Dump(value="1"), Dump(value="1"), Dump(value="2")] - - def dump_fn(): - return dumps[min(len(dumps) - 1, dump_fn.i)] # type: ignore - - dump_fn.i = 0 # type: ignore - - mgr = ActionRecordingManager( - root, - flow_name="t", - surface="browser", - dump_fn=lambda: Dump(value="1"), - stability_timeout_sec=0.4, - ) - mgr.start(entry_dump=Dump(value="1")) - entry_states = list(mgr.action_flow.states) - self.assertEqual(len(entry_states), 1) - - # Heartbeat must not create new states/actions - mgr.heartbeat() - mgr.heartbeat() - self.assertEqual(len(mgr.action_flow.states), 1) - self.assertEqual(len(mgr.action_flow.actions), 0) - - # Click with stable dump - mgr.dump_fn = lambda: Dump(value="2", url="https://example.com/2") - act = mgr.handle_raw_event( - RawUserEvent( - kind="click", - surface="browser", - target={ - "role": "button", - "name": "Go", - "automation_id": "go", - "control_type": "button", - }, - ) - ) - self.assertIsNotNone(act) - assert act is not None - self.assertEqual(act.action_type, "click") - self.assertTrue(act.before_state_id) - self.assertTrue(act.after_state_id) - self.assertTrue((root / "action_flow.json").exists()) - self.assertTrue((root / "element_catalog.json").exists()) - - def test_click_dump_uses_event_origin_page(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - seen = [] - - def dump_fn(**kwargs): - seen.append(dict(kwargs)) - return Dump(url="https://example.com/detail") - - mgr = ActionRecordingManager( - Path(tmp), surface="browser", dump_fn=dump_fn, - stability_timeout_sec=0.2, - ) - mgr.start(entry_dump=Dump(url="https://example.com/list")) - act = mgr.handle_raw_event( - RawUserEvent( - kind="click", - surface="browser", - target={"role": "button", "name": "Open"}, - url="https://example.com/detail", - meta={ - "page_id": "page_detail", - "document_id": "doc_detail", - "frame_id": "frame_main", - }, - ) - ) - self.assertTrue(seen) - self.assertTrue(all(x.get("page_id") == "page_detail" for x in seen)) - self.assertIsNotNone(act) - assert act is not None - self.assertTrue(act.after_observation_id) - - def test_duplicate_state_still_persists_each_observation(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - service = SnapshotService(root) - first = service.capture_from_dump(Dump(value="same"), change_reason="entry") - second = service.capture_from_dump( - Dump(value="same"), change_reason="after_click", force_persist=True - ) - self.assertEqual(first.id, second.id) - self.assertEqual(len(service.state_ids()), 1) - self.assertEqual(len(service.observation_ids()), 2) - metas = list((root / "observations").glob("*/meta.json")) - self.assertEqual(len(metas), 2) - - def test_inspector_events_filtered(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - mgr = ActionRecordingManager(Path(tmp), surface="windows", dump_fn=None) - mgr.status = RecorderState.RECORDING - act = mgr.handle_raw_event( - RawUserEvent(kind="click", surface="windows", from_inspector=True) - ) - self.assertIsNone(act) - - def test_flow_session_processes_events_in_fifo_order(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - session = FlowSession(Path(tmp), flow_name="fifo", surface="browser") - session.manager.start() - session.manager._debounce_sec = 0.0 - session._start_event_consumer() - for name in ("first", "second", "third"): - self.assertTrue( - session._submit( - "event", - RawUserEvent( - kind="click", - surface="browser", - target={"role": "button", "name": name}, - ), - ) - ) - session._event_queue.join() - names = [] - for path in sorted((Path(tmp) / "actions").glob("*.json")): - import json - - data = json.loads(path.read_text(encoding="utf-8")) - names.append((data["sequence"], data["target_snapshot"]["name"])) - self.assertEqual( - [name for _, name in sorted(names)], ["first", "second", "third"] - ) - session.stop() - - def test_sensitive_value_absent_from_all_action_artifacts(self) -> None: - secret = "super-secret-password-987654321" - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - mgr = ActionRecordingManager(root, surface="browser", dump_fn=None) - mgr.start() - mgr.handle_raw_event( - RawUserEvent( - kind="click", - surface="browser", - target={ - "role": "textbox", - "name": "password", - "input_type": "password", - "value": secret, - }, - meta={"raw_value": secret}, - ) - ) - persisted = "\n".join( - p.read_text(encoding="utf-8", errors="ignore") - for p in root.rglob("*") - if p.is_file() - ) - self.assertNotIn(secret, persisted) - - -class TestBrowserRecorderBridge(unittest.TestCase): - def test_pointerdown_is_fallback_when_click_never_arrives(self) -> None: - seen = [] - recorder = BrowserRecorder(on_event=seen.append) - recorder.status = RecorderState.RECORDING - recorder._pointer_fallback_sec = 0.0 - recorder._emit_dom_event( - { - "event_id": "doc:down", - "kind": "pointerdown", - "page_id": "page_1", - "document_id": "doc", - "pointer_id": 1, - "button": 0, - "x": 10, - "y": 10, - "target": {"role": "link", "tag": "a"}, - "candidate_chain": [{"role": "link", "tag": "a"}], - } - ) - recorder._flush_pending_pointerdowns() - self.assertEqual(len(seen), 1) - self.assertTrue(seen[0].meta.get("pointerdown_fallback")) - - def test_click_cancels_pointerdown_fallback(self) -> None: - seen = [] - recorder = BrowserRecorder(on_event=seen.append) - recorder.status = RecorderState.RECORDING - base = { - "page_id": "page_1", "document_id": "doc", "pointer_id": 1, - "button": 0, "x": 10, "y": 10, - "target": {"role": "button", "tag": "button"}, - "candidate_chain": [{"role": "button", "tag": "button"}], - } - recorder._emit_dom_event({**base, "event_id": "doc:down", "kind": "pointerdown"}) - recorder._emit_dom_event({**base, "event_id": "doc:click", "kind": "click"}) - recorder._flush_pending_pointerdowns(force=True) - self.assertEqual([event.kind for event in seen], ["click"]) - - def test_new_page_without_opener_still_emits_navigation(self) -> None: - seen = [] - recorder = BrowserRecorder(on_event=seen.append) - recorder.status = RecorderState.RECORDING - recorder._known_page_ids.add("page_old") - payload = { - "page_id": "page_new", "opener_page_id": "", - "url": "https://example.com/detail", "events": [], - } - with patch.object(kit_browser, "call_on_each_open_page", return_value=[payload]): - recorder._tick() - recorder._flush_pending_navigations(force=True) - self.assertEqual([event.kind for event in seen], ["navigation"]) - self.assertTrue(seen[0].meta.get("new_page")) - - def test_stop_drains_bridge_events_already_received(self) -> None: - seen = [] - recorder = BrowserRecorder(on_event=seen.append) - recorder.status = RecorderState.RECORDING - recorder._receive_bound_event( - { - "event_id": "doc:last", - "kind": "click", - "target": {"role": "button", "name": "Last"}, - } - ) - recorder.stop() - self.assertEqual([event.target.get("name") for event in seen], ["Last"]) - - def test_polled_click_is_emitted_before_due_navigation(self) -> None: - seen = [] - recorder = BrowserRecorder(on_event=seen.append) - recorder.status = RecorderState.RECORDING - recorder._pending_navigations["page_1"] = { - "due": 0.0, - "from": "https://example.com/list", - "to": "https://example.com/detail", - "page_id": "page_1", - } - payload = { - "page_id": "page_1", - "url": "https://example.com/detail", - "events": [ - { - "event_id": "doc:open", - "kind": "click", - "target": {"role": "link", "name": "Open"}, - } - ], - } - with patch.object(kit_browser, "call_on_each_open_page", return_value=[payload]): - recorder._tick() - self.assertEqual([event.kind for event in seen], ["click", "navigation"]) - - def test_bound_and_poll_fallback_event_is_deduplicated(self) -> None: - seen = [] - recorder = BrowserRecorder(on_event=seen.append) - recorder.status = RecorderState.RECORDING - event = { - "event_id": "doc:1", - "document_id": "doc", - "sequence": 1, - "kind": "click", - "target": {"role": "button", "name": "Go"}, - } - recorder._emit_dom_event(dict(event)) - recorder._emit_dom_event(dict(event)) - self.assertEqual(len(seen), 1) - - def test_bridge_preserves_page_and_frame_identity(self) -> None: - class Page: - def is_closed(self): - return False - - page = Page() - frame = object() - seen = [] - with kit_browser._bridge_lock: - kit_browser._bridge_sink = seen.append - try: - kit_browser._dispatch_bridge_event( - {"page": page, "frame": frame}, {"kind": "click"} - ) - finally: - kit_browser.clear_event_bridge() - kit_browser._page_ids.clear() - kit_browser._page_refs.clear() - kit_browser._pages_by_id.clear() - kit_browser._frame_ids.clear() - kit_browser._frame_refs.clear() - self.assertEqual(len(seen), 1) - self.assertTrue(seen[0]["page_id"].startswith("page_")) - self.assertIn(":frame_", seen[0]["frame_id"]) - - def test_svg_hit_keeps_promoted_action_target(self) -> None: - seen = [] - recorder = BrowserRecorder(on_event=seen.append) - recorder.status = RecorderState.RECORDING - recorder._emit_dom_event( - { - "event_id": "doc:svg", - "kind": "click", - "page_id": "page_video", - "target": {"role": "button", "name": "Play", "tag": "button"}, - "hit_target": {"role": "path", "tag": "path"}, - "action_target": {"role": "button", "name": "Play", "tag": "button"}, - } - ) - self.assertEqual(seen[0].target.get("role"), "button") - self.assertEqual(seen[0].meta.get("hit_target", {}).get("tag"), "path") - self.assertEqual(seen[0].meta.get("page_id"), "page_video") - - def test_navigation_failure_is_not_swallowed(self) -> None: - class Page: - url = "about:blank" - - def goto(self, *args, **kwargs): - raise RuntimeError("network down") - - with self.assertRaisesRegex(RuntimeError, "ERROR:NAVIGATION_FAILED"): - kit_browser._goto_safe(Page(), "https://example.invalid") - - -class TestWeComAdapter(unittest.TestCase): - def test_profiles(self) -> None: - self.assertEqual( - wecom_adapter.wake_profile(class_name="WeWorkWindow"), "wecom" - ) - self.assertEqual( - wecom_adapter.wake_profile(class_name="SearchResultWindow2"), - "wecom_search", - ) - self.assertEqual(resolve_wake_profile(class_name="WeWorkWindow"), "wecom") - - def test_cache_keys_isolated(self) -> None: - k1 = wecom_adapter.cache_key(profile="wecom", hwnd=100) - k2 = wecom_adapter.cache_key(profile="wecom_search", hwnd=200) - k3 = _cache_key("wecom", 100) - self.assertNotEqual(k1, k2) - self.assertEqual(k1, k3) - # Main warm key must not equal search key even with hwnd=0 confusion - self.assertNotEqual( - wecom_adapter.cache_key(profile="wecom", hwnd=1), - wecom_adapter.cache_key(profile="wecom_search", hwnd=1), - ) - - def test_proven_warm_provider_is_pinned_for_recorder_flow(self) -> None: - _caches.clear() - self.assertTrue( - note_provider_warm( - class_name="WeWorkWindow", title="WeCom", hwnd=100, findall=120 - ) - ) - report = auto_ensure_if_wecom( - class_name="WeWorkWindow", title="WeCom", hwnd=100, - findall_hint=0, - ) - self.assertTrue(report.get("ok")) - self.assertEqual(report.get("reason"), "cache_warm") - self.assertFalse(report.get("wake_applied")) - - -class TestV1FlowStillCreated(unittest.TestCase): - def test_create_flow_has_steps_and_v2(self) -> None: - import artifacts as art - - with tempfile.TemporaryDirectory() as tmp: - flow_dir = art.create_flow(Path(tmp), flow_name="demo", recording_mode="auto") - self.assertTrue((flow_dir / "steps").is_dir()) - self.assertTrue((flow_dir / "states").is_dir()) - self.assertTrue((flow_dir / "observations").is_dir()) - self.assertTrue((flow_dir / "journal").is_dir()) - self.assertTrue((flow_dir / "diagnostics").is_dir()) - flow = art.load_flow(flow_dir) - self.assertEqual(flow["schema"], "openclaw.ui_inspector.flow.v1") - self.assertIn("v2", flow) - self.assertEqual(flow["v2"]["action_flow"], "action_flow.json") - self.assertEqual(flow["v2"]["raw_event_journal"], "journal/raw_events.jsonl") - - -if __name__ == "__main__": - unittest.main() diff --git a/tools/ui_inspector/tests/unit/__init__.py b/tools/ui_inspector/tests/unit/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/tools/ui_inspector/tests/unit/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tools/ui_inspector/tests/unit/core/__init__.py b/tools/ui_inspector/tests/unit/core/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/tools/ui_inspector/tests/unit/core/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tools/ui_inspector/tests/unit/core/test_event_journal_identity.py b/tools/ui_inspector/tests/unit/core/test_event_journal_identity.py deleted file mode 100644 index 80262f0..0000000 --- a/tools/ui_inspector/tests/unit/core/test_event_journal_identity.py +++ /dev/null @@ -1,30 +0,0 @@ -import json -from pathlib import Path - -from core.pipeline.event_journal import EventJournal -from core.policies.element_identity import element_identity_key - - -def test_journal_is_append_only(tmp_path: Path) -> None: - journal = EventJournal(tmp_path) - assert journal.append({"kind": "pointerdown"}) == 1 - assert journal.append({"kind": "click"}) == 2 - rows = [json.loads(line) for line in journal.path.read_text(encoding="utf-8").splitlines()] - assert [row["journal_sequence"] for row in rows] == [1, 2] - assert [row["event"]["kind"] for row in rows] == ["pointerdown", "click"] - - -def test_focus_classes_and_query_strings_do_not_split_identity() -> None: - first = { - "role": "textbox", "tag": "input", "placeholder": "Search", - "css": "#header input.search", - } - focused = { - **first, - "class_name": "search focused hash-123", - "css": "#header input.search.focused.hash-123", - } - one = element_identity_key(first, surface="browser", page_url="https://example.com/home?a=1") - two = element_identity_key(focused, surface="browser", page_url="https://example.com/results?q=x") - assert one == two - diff --git a/tools/ui_inspector/tests/unit/core/test_target_scoring.py b/tools/ui_inspector/tests/unit/core/test_target_scoring.py deleted file mode 100644 index 666dc7f..0000000 --- a/tools/ui_inspector/tests/unit/core/test_target_scoring.py +++ /dev/null @@ -1,70 +0,0 @@ -from core.policies.target_scoring import resolve_browser_target - - -def test_huge_control_named_ancestor_never_beats_comment_hit() -> None: - hit = { - "tag": "path", "role": "path", "cursor": "pointer", - "rect": {"left": 1047, "top": 545, "right": 1070, "bottom": 568, "width": 23, "height": 23}, - "css": "#comment-input-container .send > svg > path", - } - huge = { - "tag": "div", "role": "div", "class_name": "playerControlHeight", - "rect": {"left": 160, "top": -244, "right": 1440, "bottom": 6698, "width": 1280, "height": 6942}, - "css": "#root .playerControlHeight", - } - target, scored = resolve_browser_target(hit, [hit, huge], x=1056, y=558) - assert target["tag"] == "path" - assert any("oversized_ancestor" in row["reasons"] for row in scored) - - -def test_nearest_semantic_wrapper_beats_svg_path() -> None: - hit = { - "tag": "path", "role": "path", "cursor": "pointer", - "rect": {"left": 10, "top": 10, "right": 30, "bottom": 30, "width": 20, "height": 20}, - } - wrapper = { - "tag": "span", "role": "span", "cursor": "pointer", "class_name": "sendControl", - "rect": {"left": 5, "top": 5, "right": 35, "bottom": 35, "width": 30, "height": 30}, - } - target, _ = resolve_browser_target(hit, [hit, wrapper], x=20, y=20) - assert target["class_name"] == "sendControl" - - -def test_contenteditable_root_beats_transient_inner_block() -> None: - inner = { - "tag": "div", "role": "textbox", "contenteditable": True, - "contenteditable_attr": False, - "rect": {"left": 10, "top": 10, "right": 300, "bottom": 35, "width": 290, "height": 25}, - } - root = { - "tag": "div", "role": "combobox", "explicit_role": "combobox", - "contenteditable": True, "contenteditable_attr": True, - "rect": {"left": 10, "top": 10, "right": 300, "bottom": 35, "width": 290, "height": 25}, - } - target, _ = resolve_browser_target(inner, [inner, root], x=20, y=20) - assert target["contenteditable_attr"] is True - - -def test_unique_clickable_wrapper_beats_ambiguous_graphic_leaf() -> None: - rect_path = {"left": 1047, "top": 345, "right": 1070, "bottom": 368, "width": 23, "height": 23} - rect_wrapper = {"left": 1041, "top": 338, "right": 1077, "bottom": 374, "width": 36, "height": 36} - path = { - "tag": "path", "role": "path", "cursor": "pointer", - "rect": rect_path, "css_count": 2, - } - svg = { - "tag": "svg", "role": "svg", "cursor": "pointer", - "rect": rect_wrapper, "css_count": 1, - } - wrapper = { - "tag": "span", "role": "span", "cursor": "pointer", - "rect": rect_wrapper, "css_count": 1, - "stable_css": "#comment-form > div.controls > span", - "stable_css_count": 1, - } - target, scored = resolve_browser_target( - path, [path, svg, wrapper], x=1055, y=352 - ) - assert target["tag"] == "span" - assert any("graphic_leaf" in row["reasons"] for row in scored) - assert any("event_time_unique_locator" in row["reasons"] for row in scored) diff --git a/tools/ui_inspector/tests/unit/test_app_layout.py b/tools/ui_inspector/tests/unit/test_app_layout.py deleted file mode 100644 index 4feaf18..0000000 --- a/tools/ui_inspector/tests/unit/test_app_layout.py +++ /dev/null @@ -1,120 +0,0 @@ -import json -import sys -from pathlib import Path - -_HERE = Path(__file__).resolve().parents[2] -if str(_HERE) not in sys.path: - sys.path.insert(0, str(_HERE)) - -from app import ( - _DEFAULT_FLOW_NAME_SLUGS, - _default_window_geometry, - _flow_name_for_mode, - _format_geometry, - _format_live_action, - _merge_current_live_action, -) -import artifacts as art -import target -import user_prefs - - -def test_default_geometry_is_narrow_tall_and_leaves_title_bar_clear() -> None: - width, height, x, y = _default_window_geometry((0, 0, 1920, 1040)) - - assert (width, height) == (576, 951) - assert (x, y) == (1328, 73) - assert y >= 64 - assert x + width == 1920 - 16 - - -def test_default_geometry_supports_small_and_left_side_monitors() -> None: - width, height, x, y = _default_window_geometry( - (-1366, 0, 0, 728), side="left" - ) - - assert (width, height) == (560, 648) - assert (x, y) == (-1350, 64) - assert _format_geometry(width, height, x, y) == "560x648-1350+64" - - -def test_flow_name_defaults_follow_pick_mode() -> None: - assert _flow_name_for_mode(target.SURFACE_BROWSER) == "browser" - assert _flow_name_for_mode(target.SURFACE_WINDOWS) == "desktop" - assert _flow_name_for_mode(target.SURFACE_ANDROID) == "mobile" - assert _DEFAULT_FLOW_NAME_SLUGS == frozenset({"browser", "desktop", "mobile"}) - - -def test_local_ts_is_compact_yyyymmddhhmmss() -> None: - stamp = art._local_ts() - assert len(stamp) == 14 - assert stamp.isdigit() - assert "T" not in stamp - assert "+" not in stamp - - -def test_create_flow_dir_uses_compact_timestamp(tmp_path: Path) -> None: - flow_dir = art.create_flow(tmp_path, flow_name="browser") - name = flow_dir.name - assert name.endswith("_browser") - stamp = name[: -len("_browser")] - assert len(stamp) == 14 - assert stamp.isdigit() - - -def test_hidden_legacy_recording_preference_migrates_to_auto( - tmp_path: Path, monkeypatch -) -> None: - prefs_path = tmp_path / "user_prefs.json" - prefs_path.write_text( - json.dumps( - { - "mode": "browser", - "start_url": "https://example.com", - "recording_mode": "legacy_interval", - } - ), - encoding="utf-8", - ) - monkeypatch.setattr(user_prefs, "_PREFS_PATH", prefs_path) - - assert user_prefs.load_prefs()["recording_mode"] == "auto" - - -def test_live_action_formatter_is_surface_neutral_and_two_phase() -> None: - text = _format_live_action( - { - "sequence": 3, - "action_type": "click", - "phase": "captured", - "surface": "browser", - "target_name": "提交", - "target_type": "button", - } - ) - assert "#3 点击" in text - assert "browser" in text - assert "证据采集中" in text - assert "等待异步证据" in text - - -def test_current_live_action_overwrites_without_allowing_late_evidence_to_regress() -> None: - first = {"id": "action-1", "sequence": 1, "phase": "captured"} - first_evidence = {"id": "action-1", "sequence": 1, "phase": "evidence"} - second = {"id": "action-2", "sequence": 2, "phase": "captured"} - - current = _merge_current_live_action(None, first) - current = _merge_current_live_action(current, first_evidence) - assert current == first_evidence - - current = _merge_current_live_action(current, second) - assert current == second - - current = _merge_current_live_action(current, first_evidence) - assert current == second - - -def test_current_live_action_ignores_updates_without_an_action_id() -> None: - current = {"id": "action-2", "sequence": 2, "phase": "captured"} - - assert _merge_current_live_action(current, {"sequence": 3}) is current diff --git a/tools/ui_inspector/ui/__init__.py b/tools/ui_inspector/ui/__init__.py deleted file mode 100644 index 8acdbd3..0000000 --- a/tools/ui_inspector/ui/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""User-interface package; legacy app.py remains the compatibility entry.""" - diff --git a/tools/ui_inspector/ui/controller.py b/tools/ui_inspector/ui/controller.py deleted file mode 100644 index 72115a3..0000000 --- a/tools/ui_inspector/ui/controller.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Application-facing session facade. - -The Tk view remains in app.py during the compatibility migration; new callers -should depend on this facade instead of constructing recorder internals. -""" - -from flow_session import FlowSession - -__all__ = ["FlowSession"] - diff --git a/tools/ui_inspector/user_prefs.py b/tools/ui_inspector/user_prefs.py deleted file mode 100644 index 1c19830..0000000 --- a/tools/ui_inspector/user_prefs.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Persist UI Inspector prefs (pick mode + start URL + recording mode) across runs.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - -import target - -_HERE = Path(__file__).resolve().parent -_PREFS_PATH = _HERE / "user_prefs.json" - -DEFAULT_MODE = target.SURFACE_BROWSER -# 起始 URL：默认空；仅记住用户主动填写过的值 -DEFAULT_START_URL = "" -DEFAULT_RECORDING_MODE = "auto" - -_VALID_MODES = { - target.SURFACE_BROWSER, - target.SURFACE_WINDOWS, - target.SURFACE_ANDROID, # UI label「手机」 -} - -_VALID_RECORDING = {"auto", "guided", "checkpoint", "legacy_interval"} - - -def prefs_path() -> Path: - return _PREFS_PATH - - -def load_prefs() -> dict[str, Any]: - raw: dict[str, Any] = {} - try: - if _PREFS_PATH.is_file(): - data = json.loads(_PREFS_PATH.read_text(encoding="utf-8")) - if isinstance(data, dict): - raw = data - except Exception: - raw = {} - mode = str(raw.get("mode") or DEFAULT_MODE).strip().lower() - if mode in {"mobile", "phone", "ios"}: - mode = target.SURFACE_ANDROID - if mode not in _VALID_MODES: - mode = DEFAULT_MODE - # 允许空字符串；未写过 start_url 键时也视为空 - if "start_url" in raw: - start_url = str(raw.get("start_url") or "").strip() - else: - start_url = DEFAULT_START_URL - # v0.14.3 standardizes the public UI on automatic action recording. - # Older values remain accepted by lower-level capture APIs, but must not - # silently put the simplified UI into a hidden legacy mode. - recording_mode = DEFAULT_RECORDING_MODE - return { - "mode": mode, - "start_url": start_url, - "recording_mode": recording_mode, - } - - -def save_prefs( - *, - mode: str, - start_url: str, - recording_mode: str = DEFAULT_RECORDING_MODE, -) -> None: - mode_n = (mode or DEFAULT_MODE).strip().lower() - if mode_n in {"mobile", "phone", "ios"}: - mode_n = target.SURFACE_ANDROID - if mode_n not in _VALID_MODES: - mode_n = DEFAULT_MODE - url = (start_url or "").strip() - rec = (recording_mode or DEFAULT_RECORDING_MODE).strip().lower() - if rec not in _VALID_RECORDING: - rec = DEFAULT_RECORDING_MODE - payload = { - "mode": mode_n, - "start_url": url, - "recording_mode": rec, - } - try: - _PREFS_PATH.write_text( - json.dumps(payload, ensure_ascii=False, indent=2) + "\n", - encoding="utf-8", - ) - except Exception: - pass