"""Tk panel v0.14: Action-driven UI Automation Recorder (compatible with v1 dumps). 拾取模式（单选，记忆上次）：浏览器 / 桌面 / 手机。 - 浏览器：强制 Playwright；起始 URL 自动打开；Browser Recorder - 桌面：强制 UIA；Windows Recorder（进程外轮询，无注入） - 手机：占位未实现 录制模式：自动录制（默认）/ 引导录制 / 手动检查点；legacy 间隔 dump 可选。 """ from __future__ import annotations import argparse import ctypes import os import queue import subprocess import sys import threading from ctypes import wintypes from pathlib import Path import tkinter as tk from tkinter import messagebox, simpledialog, ttk _HERE = Path(__file__).resolve().parent if str(_HERE) not in sys.path: sys.path.insert(0, str(_HERE)) import artifacts as art # noqa: E402 import desktop_uia as uia # noqa: E402 import kit_browser # noqa: E402 import skip_policy # noqa: E402 import target # noqa: E402 import user_prefs # noqa: E402 from surfaces.registry import get_driver # noqa: E402 from flow_session import FlowSession # noqa: E402 from drivers.base import DumpResult # noqa: E402 try: from . import __version__ except Exception: try: import importlib.util _spec = importlib.util.spec_from_file_location( "ui_inspector_pkg", _HERE / "__init__.py" ) _mod = importlib.util.module_from_spec(_spec) # type: ignore[arg-type] assert _spec and _spec.loader _spec.loader.exec_module(_mod) __version__ = getattr(_mod, "__version__", "0.16.0") except Exception: __version__ = "0.16.0" # Legacy interval mode: after each dump finishes, wait this long, then dump again. _AUTO_GAP_MS = 2000 # Heartbeat for action-driven mode (flush idle input; do NOT create steps). _HEARTBEAT_MS = 800 # After frontmost target is resolved, short delay before first dump. _AUTO_START_DELAY_MS = 400 # After「开始流程」with auto-pick: countdown so user can focus the target app. _START_COUNTDOWN_SEC = 5 _FRONTMOST_WAIT_SEC = 15.0 _STABILITY_TIMEOUT_SEC = 3.0 # Recording modes _MODE_AUTO = "auto" _MODE_GUIDED = "guided" _MODE_CHECKPOINT = "checkpoint" _MODE_LEGACY = "legacy_interval" _WINDOW_RIGHT_MARGIN = 16 _WINDOW_BOTTOM_MARGIN = 16 _WINDOW_MIN_WIDTH = 520 _WINDOW_MIN_HEIGHT = 600 # Default flow-name slugs aligned with the pick-mode radio labels. # Switching mode only auto-updates the field when it still holds one of these. _MODE_FLOW_NAMES = { target.SURFACE_BROWSER: "browser", target.SURFACE_WINDOWS: "desktop", target.SURFACE_ANDROID: "mobile", } _DEFAULT_FLOW_NAME_SLUGS = frozenset(_MODE_FLOW_NAMES.values()) def _flow_name_for_mode(mode: str) -> str: return _MODE_FLOW_NAMES.get(mode, "desktop") def _clamp(value: int, low: int, high: int) -> int: return max(low, min(high, value)) def _default_window_geometry( work_area: tuple[int, int, int, int], *, side: str = "right", ) -> tuple[int, int, int, int]: """Return a narrow console geometry inside a monitor work area. The top gap deliberately leaves the target application's title-bar controls unobstructed. Coordinates may be negative for monitors left of the primary. """ left, top, right, bottom = work_area work_width = max(1, right - left) work_height = max(1, bottom - top) width = _clamp(round(work_width * 0.30), 560, 620) width = min(width, max(420, work_width - 32)) top_gap = _clamp(round(work_height * 0.07), 64, 88) height = max(480, work_height - top_gap - _WINDOW_BOTTOM_MARGIN) x = ( left + _WINDOW_RIGHT_MARGIN if side == "left" else right - width - _WINDOW_RIGHT_MARGIN ) y = top + top_gap return width, height, x, y def _format_geometry(width: int, height: int, x: int, y: int) -> str: x_part = f"+{x}" if x >= 0 else str(x) y_part = f"+{y}" if y >= 0 else str(y) return f"{width}x{height}{x_part}{y_part}" def _format_live_action(item: dict) -> str: """Compact, surface-neutral live action text for the result panel.""" labels = { "click": "点击", "fill": "输入", "type": "输入", "press_key": "按键", "navigation": "导航", "checkpoint": "检查点", } phases = { "captured": "已捕获，证据采集中", "evidence": "证据已更新", "evidence_failed": "证据采集失败", } action_type = str(item.get("action_type") or "unknown") lines = [ f"#{item.get('sequence')} {labels.get(action_type, action_type)} · " f"{phases.get(str(item.get('phase') or ''), str(item.get('phase') or ''))}", f" 表面: {item.get('surface') or '-'} 目标: " f"{item.get('target_name') or '-'} ({item.get('target_type') or '-'})", ] if item.get("input_summary"): lines.append(f" 输入: {item['input_summary']}") if item.get("window_class"): lines.append( f" 窗口: {item.get('window_title') or '-'} [{item.get('window_class')}]" ) if item.get("locator"): lines.append( f" 定位: {item['locator']} match={item.get('match_count')} " f"unique={item.get('unique')}" ) elif item.get("phase") == "evidence_failed": lines.append(f" 错误: {item.get('error') or '未知错误'}") else: lines.append(" 定位: 等待异步证据") return "\n".join(lines) def _merge_current_live_action(current: dict | None, incoming: dict) -> dict | None: """Keep the newest action while allowing its evidence phase to replace it.""" if not str(incoming.get("id") or ""): return current if current is None: return dict(incoming) current_sequence = int(current.get("sequence") or 0) incoming_sequence = int(incoming.get("sequence") or 0) if incoming_sequence < current_sequence: return current return dict(incoming) class InspectorApp: def __init__( self, *, captures_root: Path, scene: str = "", max_depth: int = 16, max_nodes: int = 8000, ) -> None: self.captures_root = captures_root self.scene = scene self.max_depth = max_depth self.max_nodes = max_nodes self.flow_dir: Path | None = None self.flow: dict | None = None self.last_window: dict | None = None self.sticky_pid: int | None = None # Browser entries use pid; native entries use (pid, window class) so a # WeCom SearchResultWindow2 is not mistaken for its already-warmed main # WeWorkWindow merely because both belong to the same process. self._warmed_pids: set[object] = set() self._busy = False self._auto_loop_active = False self._pending_end = False self._auto_after_id: str | None = None self._countdown_after_id: str | None = None self._countdown_left = 0 self._session: FlowSession | None = None self._desktop_entry_dump: DumpResult | None = None self._heartbeat_after_id: str | None = None self._rec_paused = False self._current_live_action: dict | None = None self._ui_queue: queue.Queue[tuple[object, tuple, dict]] = queue.Queue() self._ui_poll_after_id: str | None = None self._closed = False prefs = user_prefs.load_prefs() initial_mode = str(prefs.get("mode") or user_prefs.DEFAULT_MODE) if initial_mode in {"mobile", "phone", "ios"}: initial_mode = target.SURFACE_ANDROID if initial_mode not in { target.SURFACE_BROWSER, target.SURFACE_WINDOWS, target.SURFACE_ANDROID, }: initial_mode = user_prefs.DEFAULT_MODE scene_name = (scene or "").strip() initial_flow_name = scene_name or _flow_name_for_mode(initial_mode) self.root = tk.Tk() self.root.title(f"OpenClaw UI Inspector v{__version__}") self._dock_side = "right" self._apply_default_geometry() self.root.attributes("-topmost", True) self.flow_name_var = tk.StringVar(value=initial_flow_name) self.mode_var = tk.StringVar(value=initial_mode) self.start_url_var = tk.StringVar(value=str(prefs.get("start_url") or "")) # Standard UI always records actions automatically. Legacy strategies # remain supported internally for old captures and developer tooling. self.auto_pick_var = tk.BooleanVar(value=True) self.recording_mode_var = tk.StringVar(value=_MODE_AUTO) # 默认开：悬停即可采搜索浮层，不必点进会话；均可随时取消勾选回退旧行为 self.under_cursor_var = tk.BooleanVar(value=True) self.prefer_wecom_search_var = tk.BooleanVar(value=True) self.flow_var = tk.StringVar(value="流程：未开始（请先点「开始流程」）") self.target_var = tk.StringVar(value="等待开始流程") self.rec_stats_var = tk.StringVar( value="recorder: idle | actions=0 states=0 observations=0 elements=0" ) self.status_var = tk.StringVar( value="选择模式后点击「开始流程」。" ) self._build_ui() for var in ( self.flow_var, self.rec_stats_var, self.target_var, self.status_var, ): var.trace_add("write", self._on_status_var_changed) self._render_status_summary() self._refresh_mode_widgets() self.root.protocol("WM_DELETE_WINDOW", self._on_close) self._schedule_ui_queue_poll() def _current_monitor_work_area(self) -> tuple[int, int, int, int]: """Get the work area of the monitor containing the mouse pointer.""" if sys.platform == "win32": try: class MONITORINFO(ctypes.Structure): _fields_ = [ ("cbSize", wintypes.DWORD), ("rcMonitor", wintypes.RECT), ("rcWork", wintypes.RECT), ("dwFlags", wintypes.DWORD), ] point = wintypes.POINT() user32 = ctypes.windll.user32 user32.GetCursorPos.argtypes = [ctypes.POINTER(wintypes.POINT)] user32.GetCursorPos.restype = wintypes.BOOL user32.MonitorFromPoint.argtypes = [wintypes.POINT, wintypes.DWORD] user32.MonitorFromPoint.restype = wintypes.HANDLE user32.GetMonitorInfoW.argtypes = [ wintypes.HANDLE, ctypes.POINTER(MONITORINFO), ] user32.GetMonitorInfoW.restype = wintypes.BOOL if user32.GetCursorPos(ctypes.byref(point)): monitor = user32.MonitorFromPoint(point, 2) info = MONITORINFO() info.cbSize = ctypes.sizeof(info) if monitor and user32.GetMonitorInfoW(monitor, ctypes.byref(info)): area = info.rcWork return area.left, area.top, area.right, area.bottom except Exception: pass return 0, 0, self.root.winfo_screenwidth(), self.root.winfo_screenheight() def _apply_default_geometry(self) -> None: width, height, x, y = _default_window_geometry( self._current_monitor_work_area(), side=self._dock_side ) self.root.geometry(_format_geometry(width, height, x, y)) self.root.minsize(min(_WINDOW_MIN_WIDTH, width), min(_WINDOW_MIN_HEIGHT, height)) def _toggle_dock(self) -> None: self._dock_side = "left" if self._dock_side == "right" else "right" self._apply_default_geometry() self.dock_btn.configure(text="靠右" if self._dock_side == "left" else "靠左") def _on_status_var_changed(self, *_args: object) -> None: self._render_status_summary() def _render_status_summary(self) -> None: if not hasattr(self, "status_text"): return if not self.flow: headline = "● 未开始 · recorder idle" else: v2 = self.flow.get("v2") or {} live = self._session.stats() if self._session else {} status = str(self.flow.get("status") or "unknown") status_name = {"open": "录制中", "ended": "已结束"}.get(status, status) action_count = live.get("action_count", v2.get("action_count", 0)) state_count = live.get("state_count", v2.get("state_count", 0)) dropped = live.get("dropped_events", 0) input_health = live.get("input_backend") or {} input_suffix = "" if input_health: input_suffix = ( f" · input {input_health.get('raw_input_events', 0)}" f" · {input_health.get('backend', 'unknown')}" ) headline = ( f"● {status_name} · actions {action_count} · states {state_count}" f" · dropped {dropped}{input_suffix}" ) target_line = self.target_var.get().strip() status_line = self.status_var.get().strip() lines = [headline] if target_line: lines.append(target_line) if status_line and status_line != target_line: lines.append(status_line) self.status_text.configure(state=tk.NORMAL) self.status_text.delete("1.0", tk.END) self.status_text.insert(tk.END, "\n".join(line for line in lines if line)) self.status_text.configure(state=tk.DISABLED) def _post_ui(self, callback, *args, **kwargs) -> None: """Worker-safe UI handoff; workers must never call Tk directly.""" self._ui_queue.put((callback, args, kwargs)) def _schedule_ui_queue_poll(self) -> None: self._ui_poll_after_id = self.root.after(40, self._drain_ui_queue) def _drain_ui_queue(self) -> None: self._ui_poll_after_id = None for _ in range(100): try: callback, args, kwargs = self._ui_queue.get_nowait() except queue.Empty: break try: callback(*args, **kwargs) # type: ignore[operator] except Exception as exc: try: self._set_status(f"后台结果处理失败: {exc}") except Exception: pass try: if self.root.winfo_exists(): self._schedule_ui_queue_poll() except Exception: pass def _current_mode(self) -> str: mode = (self.mode_var.get() or user_prefs.DEFAULT_MODE).strip().lower() if mode in {"mobile", "phone", "ios"}: return target.SURFACE_ANDROID if mode not in { target.SURFACE_BROWSER, target.SURFACE_WINDOWS, target.SURFACE_ANDROID, }: return user_prefs.DEFAULT_MODE return mode def _persist_prefs(self) -> None: user_prefs.save_prefs( mode=self._current_mode(), start_url=(self.start_url_var.get() or "").strip(), recording_mode=_MODE_AUTO, ) def _current_recording_mode(self) -> str: return _MODE_AUTO def _is_legacy_interval(self) -> bool: return self._current_recording_mode() == _MODE_LEGACY def _on_mode_change(self) -> None: self._persist_prefs() self._sync_default_flow_name() self._refresh_mode_widgets() mode = self._current_mode() if mode == target.SURFACE_BROWSER: self._set_status( "已选「浏览器自动化」：开始流程后将打开起始 URL 并 dump Playwright 页。" ) elif mode == target.SURFACE_WINDOWS: self._set_status( "已选「桌面自动化」：开始流程后按光标/前台窗口做 UIA dump（不启浏览器）。" ) else: self._set_status("已选「手机自动化」：尚未实现，开始流程时会提示。") def _sync_default_flow_name(self) -> None: """Keep flow name aligned with pick mode unless the user customized it.""" current = (self.flow_name_var.get() or "").strip() if current and current not in _DEFAULT_FLOW_NAME_SLUGS: return self.flow_name_var.set(_flow_name_for_mode(self._current_mode())) def _refresh_mode_widgets(self) -> None: mode = self._current_mode() is_browser = mode == target.SURFACE_BROWSER is_desktop = mode == target.SURFACE_WINDOWS try: if is_browser: self.url_row.grid() else: self.url_row.grid_remove() except Exception: pass try: if is_desktop: self.desktop_options.pack( fill=tk.X, padx=8, pady=(0, 4), before=self.result_panel, ) else: self.desktop_options.pack_forget() except Exception: pass def _synthetic_browser_info(self) -> uia.WindowInfo: return uia.WindowInfo( hwnd=0, pid=0, title="Playwright", class_name="PlaywrightPage", process_name="chrome.exe", ) def _build_ui(self) -> None: pad = {"padx": 8, "pady": (4, 2)} top = ttk.Frame(self.root) top.pack(fill=tk.X, padx=8, pady=(8, 2)) top.columnconfigure(1, weight=1) ttk.Label(top, text="流程名").grid(row=0, column=0, sticky=tk.W) ttk.Entry(top, textvariable=self.flow_name_var).grid( row=0, column=1, sticky=tk.EW, padx=(4, 6) ) ttk.Button(top, text="打开目录", command=self.on_open_out).grid( row=0, column=2, padx=(0, 4) ) self.dock_btn = ttk.Button(top, text="靠左", width=6, command=self._toggle_dock) self.dock_btn.grid(row=0, column=3) mode_box = ttk.LabelFrame(self.root, text="拾取模式") mode_box.pack(fill=tk.X, **pad) mode_box.columnconfigure(0, weight=1) mode_row = ttk.Frame(mode_box) mode_row.grid(row=0, column=0, sticky=tk.EW, padx=6, pady=(2, 0)) for column in range(3): mode_row.columnconfigure(column, weight=1) ttk.Radiobutton( mode_row, text="浏览器自动化", value=target.SURFACE_BROWSER, variable=self.mode_var, command=self._on_mode_change, ).grid(row=0, column=0, sticky=tk.W, padx=4) ttk.Radiobutton( mode_row, text="桌面自动化", value=target.SURFACE_WINDOWS, variable=self.mode_var, command=self._on_mode_change, ).grid(row=0, column=1, sticky=tk.W, padx=4) ttk.Radiobutton( mode_row, text="手机自动化", value=target.SURFACE_ANDROID, variable=self.mode_var, command=self._on_mode_change, ).grid(row=0, column=2, sticky=tk.W, padx=4) self.url_row = ttk.Frame(mode_box) self.url_row.grid(row=1, column=0, sticky=tk.EW, padx=6, pady=(2, 4)) self.url_row.columnconfigure(1, weight=1) ttk.Label(self.url_row, text="起始 URL").grid(row=0, column=0, sticky=tk.W) ttk.Entry(self.url_row, textvariable=self.start_url_var).grid( row=0, column=1, sticky=tk.EW, padx=(6, 0) ) btns = ttk.LabelFrame(self.root, text="流程控制") btns.pack(fill=tk.X, **pad) for column in range(3): btns.columnconfigure(column, weight=1, uniform="flow_buttons") self.start_btn = ttk.Button(btns, text="开始", command=self.on_start_flow) self.start_btn.grid(row=0, column=0, sticky=tk.EW, padx=(6, 2), pady=(3, 6)) self.pause_btn = ttk.Button(btns, text="暂停", command=self.on_pause_resume) self.pause_btn.grid(row=0, column=1, sticky=tk.EW, padx=2, pady=(3, 6)) self.end_btn = ttk.Button(btns, text="结束", command=self.on_end_flow) self.end_btn.grid(row=0, column=2, sticky=tk.EW, padx=(2, 6), pady=(3, 6)) self.desktop_options = ttk.LabelFrame(self.root, text="桌面拾取选项") self.under_cursor_chk = ttk.Checkbutton( self.desktop_options, text="光标下优先（悬停、不点击）", variable=self.under_cursor_var, ) self.under_cursor_chk.pack(anchor=tk.W, padx=8, pady=(2, 0)) self.wecom_chk = ttk.Checkbutton( self.desktop_options, text="企微搜索浮层优先（SearchResultWindow2）", variable=self.prefer_wecom_search_var, ) self.wecom_chk.pack(anchor=tk.W, padx=8, pady=(0, 4)) self.result_panel = ttk.LabelFrame(self.root, text="运行状态与拾取结果") self.result_panel.pack(fill=tk.BOTH, expand=True, padx=8, pady=(4, 8)) self.status_text = tk.Text( self.result_panel, height=3, wrap=tk.NONE, borderwidth=0, padx=5, pady=3, ) self.status_text.pack(fill=tk.X, padx=5, pady=(3, 2)) self.status_text.configure(state=tk.DISABLED) ttk.Separator(self.result_panel, orient=tk.HORIZONTAL).pack( fill=tk.X, padx=5, pady=(0, 3) ) detail_box = ttk.Frame(self.result_panel) detail_box.pack(fill=tk.BOTH, expand=True, padx=3, pady=(0, 3)) scroll = ttk.Scrollbar(detail_box) scroll.pack(side=tk.RIGHT, fill=tk.Y) self.detail = tk.Text( detail_box, height=10, wrap=tk.WORD, borderwidth=0, yscrollcommand=scroll.set, ) self.detail.pack(fill=tk.BOTH, expand=True, padx=(3, 0), pady=3) scroll.config(command=self.detail.yview) self.detail.insert( tk.END, "1. 选择拾取模式\n" "2. 点击「开始流程」，然后操作目标窗口\n" "3. 完成后点击「结束流程」\n\n" "动作、元素定位和诊断结果会显示在这里。\n", ) self.detail.configure(state=tk.DISABLED) self._refresh_button_states() def _flow_is_open(self) -> bool: return bool(self.flow_dir and self.flow and self.flow.get("status") == "open") def _in_start_countdown(self) -> bool: return self._countdown_after_id is not None or self._countdown_left > 0 def _on_recording_mode_change(self) -> None: self._persist_prefs() mode = self._current_recording_mode() if mode == _MODE_LEGACY: self._set_status("已选 Legacy 间隔 dump：将按约 2 秒串行写 steps（兼容旧行为）。") elif mode == _MODE_GUIDED: self._set_status("已选引导录制：每捕获一个动作后在本框展示定位与后置条件建议。") elif mode == _MODE_CHECKPOINT: self._set_status("已选手动检查点：请用「采集界面检查点」；不会自动录用户操作。") self.auto_pick_var.set(False) else: self._set_status("已选自动录制：Recorder 捕获用户操作并双写 v1/v2 资产。") def _refresh_button_states(self) -> None: open_flow = self._flow_is_open() counting = self._in_start_countdown() self.end_btn.configure(state=tk.NORMAL if open_flow else tk.DISABLED) self.start_btn.configure( state=tk.DISABLED if (self._busy or counting or open_flow) else tk.NORMAL ) try: self.pause_btn.configure( state=( tk.NORMAL if (open_flow and self._session and not self._busy and not counting) else tk.DISABLED ) ) except Exception: pass def _update_rec_stats(self) -> None: if not self._session: self.rec_stats_var.set( "recorder: idle | actions=0 states=0 observations=0 elements=0" ) return st = self._session.stats() input_health = st.get("input_backend") or {} input_summary = "" if input_health: input_summary = ( f" input={input_health.get('backend', 'unknown')}" f" raw={input_health.get('raw_input_events', 0)}" f" mouse={input_health.get('mouse_events', 0)}" f" keyboard={input_health.get('keyboard_events', 0)}" f" input_dropped={input_health.get('queue_dropped', 0)}" f" input_error={input_health.get('last_error') or '-'} |" ) self.rec_stats_var.set( f"recorder={st.get('recorder_status')}/{st.get('surface_recorder')} | " f"actions={st.get('action_count')} states={st.get('state_count')} " f"observations={st.get('observation_count')} elements={st.get('element_count')} " f"queue={st.get('queued_events', 0)} dropped={st.get('dropped_events', 0)} | " f"{input_summary} " f"last={st.get('last_action_type') or '-'} " f"target={st.get('last_target') or '-'} " f"page={st.get('last_page_id') or '-'} " f"obs={st.get('last_observation_id') or '-'} " f"match={st.get('last_match_count')} unique={st.get('last_unique')}" ) def _update_flow_label(self) -> None: if not self.flow_dir or not self.flow: self.flow_var.set("流程：未开始（请先点「开始流程」）") return if self._auto_loop_active: auto = "LOOP" if self._is_legacy_interval() else "REC" elif self.auto_pick_var.get() and self._flow_is_open(): auto = "armed" else: auto = "off" v2 = (self.flow or {}).get("v2") or {} live = self._session.stats() if self._session else {} action_count = live.get("action_count", v2.get("action_count", 0)) state_count = live.get("state_count", v2.get("state_count", 0)) observation_count = live.get( "observation_count", v2.get("observation_count", 0) ) self.flow_var.set( f"流程：{self.flow.get('flow_name')} | mode={self._current_mode()} | " f"rec={self._current_recording_mode()} | " f"status={self.flow.get('status')} | " f"legacy_steps={self.flow.get('step_count')} " f"actions={action_count} states={state_count} " f"observations={observation_count} | auto={auto} | {self.flow_dir}" ) self._update_rec_stats() def _set_status(self, text: str) -> None: self.status_var.set(text) def _set_detail(self, text: str) -> None: self.detail.configure(state=tk.NORMAL) self.detail.delete("1.0", tk.END) self.detail.insert(tk.END, text) self.detail.see("1.0") self.detail.configure(state=tk.DISABLED) def _on_session_action_update(self, action, phase: str) -> None: """Snapshot an ActionRecord on a worker, then hand it safely to Tk.""" target_snapshot = dict(getattr(action, "target_snapshot", {}) or {}) evidence = dict(getattr(action, "evidence", {}) or {}) input_payload = dict(getattr(action, "input", {}) or {}) scope = dict(target_snapshot.get("window_scope") or {}) input_summary = "" if input_payload.get("key"): input_summary = str(input_payload.get("key")) elif input_payload.get("text"): value = str(input_payload.get("text")) input_summary = value if len(value) <= 80 else value[:77] + "..." item = { "id": str(getattr(action, "id", "") or ""), "sequence": int(getattr(action, "sequence", 0) or 0), "action_type": str(getattr(action, "action_type", "") or ""), "surface": str(getattr(action, "surface", "") or ""), "phase": phase, "target_name": target_snapshot.get("name") or target_snapshot.get("automation_id") or target_snapshot.get("semantic_name") or target_snapshot.get("role") or target_snapshot.get("tag") or "-", "target_type": target_snapshot.get("control_type") or target_snapshot.get("role") or target_snapshot.get("tag") or "-", "window_title": scope.get("title") or "", "window_class": scope.get("class_name") or "", "input_summary": input_summary, "locator": evidence.get("primary_locator") or "", "match_count": evidence.get("locator_match_count"), "unique": evidence.get("locator_unique"), "error": evidence.get("eventual_evidence_error") or evidence.get("error") or "", } self._post_ui(self._apply_live_action_update, item) def _apply_live_action_update(self, item: dict) -> None: updated = _merge_current_live_action(self._current_live_action, item) if updated is self._current_live_action: return self._current_live_action = updated assert updated is not None self.target_var.set( f"当前动作：#{updated.get('sequence')} {updated.get('action_type')} · " f"{updated.get('phase')}" ) self._set_detail("当前步骤\n\n" + _format_live_action(updated)) def _show_picking_placeholder(self, *, auto: bool) -> None: n = int((self.flow or {}).get("step_count") or 0) + 1 mode = "自动" if auto else "检查点" self.target_var.set(f"{mode}采集中…（即将写入第 {n} 步）") self._set_status(f"正在{mode}采集：已清空本框上次结果，请稍候…") self._set_detail( "（已清空上次本步结果）\n\n" f"正在读取控件树，完成后写入当前流程第 {n} 步…\n" ) def _exclude_hwnds(self) -> set[int]: try: return {int(self.root.winfo_id())} except Exception: return set() def _cancel_auto_timer(self) -> None: if self._auto_after_id is not None: try: self.root.after_cancel(self._auto_after_id) except Exception: pass self._auto_after_id = None def _cancel_countdown(self) -> None: if self._countdown_after_id is not None: try: self.root.after_cancel(self._countdown_after_id) except Exception: pass self._countdown_after_id = None self._countdown_left = 0 def _cancel_heartbeat(self) -> None: if self._heartbeat_after_id is not None: try: self.root.after_cancel(self._heartbeat_after_id) except Exception: pass self._heartbeat_after_id = None def _stop_auto_loop(self) -> None: self._auto_loop_active = False self._cancel_auto_timer() self._cancel_countdown() self._cancel_heartbeat() def _stop_session(self) -> None: if self._session is not None: try: self._session.stop() except Exception: pass self._session = None self._rec_paused = False try: self.pause_btn.configure(text="暂停") except Exception: pass def _schedule_heartbeat(self) -> None: self._cancel_heartbeat() if not self._auto_loop_active or not self._flow_is_open(): return if self._is_legacy_interval(): return self._heartbeat_after_id = self.root.after(_HEARTBEAT_MS, self._heartbeat_tick) def _heartbeat_tick(self) -> None: self._heartbeat_after_id = None if not self._auto_loop_active or not self._flow_is_open(): return if self._session and not self._rec_paused: try: self._session.heartbeat() except Exception: pass self._update_flow_label() # Guided mode: refresh last action detail if self._current_recording_mode() == _MODE_GUIDED: self._show_guided_last_action() self._schedule_heartbeat() def _show_guided_last_action(self) -> None: if not self._session: return act = self._session.manager.last_action if not act: return lines = [ f"最近动作 #{act.sequence} {act.action_type} status={act.status}", f"target: {(act.target_snapshot or {}).get('name') or (act.target_snapshot or {}).get('automation_id')}", f"scope: {(act.target_snapshot or {}).get('window_scope')}", f"primary locator: {(act.evidence or {}).get('primary_locator')}", f"match count: {(act.evidence or {}).get('locator_match_count')} unique={(act.evidence or {}).get('locator_unique')}", "proposed postconditions:", ] for p in (act.proposed_postconditions or [])[:8]: lines.append(f" - [{p.get('status')}] {p.get('description')}") if self._session.manager.last_warning: lines.append(f"warning: {self._session.manager.last_warning}") self._set_detail("\n".join(lines)) def _make_dump_fn(self): """Build a dump callable for SnapshotService (runs on worker threads).""" # Snapshot every Tk-owned value on the main thread. The returned # callable is deliberately free of Tk calls so it cannot participate # in a Tk <-> recorder lock inversion. mode = self._current_mode() under_cursor = bool(self.under_cursor_var.get()) prefer_search = bool(self.prefer_wecom_search_var.get()) sticky_pid = self.sticky_pid exclude_hwnds = self._exclude_hwnds() max_depth = self.max_depth max_nodes = self.max_nodes warmed_pids = self._warmed_pids def dump_fn( *, page_id: str = "", document_id: str = "", frame_id: str = "", target_hwnd: int = 0, target_pid: int = 0, target_class_name: str = "", target_title: str = "", ): uia.ensure_com_sta() surface = mode sel_meta: dict = {"pick_mode": mode, "sticky_pid": sticky_pid} if surface == target.SURFACE_BROWSER: info = self._synthetic_browser_info() sel_meta["strategy"] = "playwright_session" elif target_hwnd: try: info = uia.window_info_from_hwnd(int(target_hwnd)) except Exception: info = None if info is None and target_class_name == "SearchResultWindow2": info = uia.find_wecom_search_result_window( pid=int(target_pid or 0) or None, exclude_hwnds=exclude_hwnds, ) if info is None: raise RuntimeError( f"event window disappeared: hwnd={target_hwnd} class={target_class_name}" ) cands = [info] sel_meta.update( { "strategy": "event_window", "chosen_hwnd": int(info.hwnd), "chosen_class": info.class_name, "event_title": target_title, } ) sel_meta["candidate_count"] = 1 else: info, cands, win_meta = uia.resolve_capture_target( exclude_hwnds=exclude_hwnds, prefer_pid=sticky_pid, under_cursor=under_cursor, prefer_wecom_search_popup=prefer_search, ) sel_meta.update(win_meta) sel_meta["candidate_count"] = len(cands) driver = get_driver(surface) warm_key = ( (int(info.pid), str(info.class_name or "")) if surface == target.SURFACE_WINDOWS else int(info.pid or 0) ) warmup_mode = "light" if warm_key in warmed_pids else "full" if surface == target.SURFACE_WINDOWS: # Recording already has a persisted event-time element atlas. # Evidence may probe the full tree, but must never run a # blocking product wake in the middle of user input. warmup_mode = "probe" if target_class_name == "SearchResultWindow2": # A search popup may live for only a few seconds. Persist its # bounded tree immediately; heavyweight kit wake/FindAll remains # supplementary and must not make the event-time snapshot miss it. warmup_mode = "off" # Recorder path: after-action dumps may warm; click capture itself never warms. dump_kwargs = dict( info=info, selection_meta=sel_meta, max_depth=(12 if target_class_name == "SearchResultWindow2" else max_depth), max_nodes=(1500 if target_class_name == "SearchResultWindow2" else max_nodes), warmup_mode=warmup_mode, warmed_pids=warmed_pids, ) if surface == target.SURFACE_BROWSER: dump_kwargs.update( page_id=page_id, document_id=document_id, frame_id=frame_id, ) result = driver.dump(**dump_kwargs) if info.pid: warmed_pids.add(warm_key) return result return dump_fn def _start_recording_session( self, *, entry_dump=None, checkpoint_only: bool = False ) -> None: if not self.flow_dir or not self.flow: return rec_mode = _MODE_CHECKPOINT if checkpoint_only else self._current_recording_mode() session = FlowSession( self.flow_dir, flow_name=str(self.flow.get("flow_name") or "flow"), surface=self._current_mode(), recording_mode=rec_mode, dump_fn=self._make_dump_fn(), exclude_hwnds=self._exclude_hwnds, on_action_update=self._on_session_action_update, stability_timeout_sec=_STABILITY_TIMEOUT_SEC, ) self._session = session if rec_mode == _MODE_LEGACY: self._auto_loop_active = True self._schedule_auto_next(_AUTO_START_DELAY_MS) return self._busy = True self._refresh_button_states() self._set_status("正在后台采集入口状态，界面仍可响应……") def work() -> None: try: if rec_mode == _MODE_CHECKPOINT: session.manager.action_flow.recording_mode = _MODE_CHECKPOINT session.manager.start(entry_dump=entry_dump) session._refresh_stats() else: session.start(entry_dump=entry_dump) stats = session.stats() if int(stats.get("state_count") or 0) == 0: warning = str(stats.get("warning") or "入口状态采集没有产物") raise RuntimeError(f"ERROR:ENTRY_CAPTURE_FAILED {warning}") except Exception as exc: self._post_ui(self._on_recording_session_started, session, rec_mode, exc) try: session.stop() except Exception: pass return self._post_ui(self._on_recording_session_started, session, rec_mode, None) threading.Thread( target=work, name="ui-inspector-session-start", daemon=True ).start() def _on_recording_session_started( self, session: FlowSession, rec_mode: str, error: BaseException | None ) -> None: if self._session is not session: return self._busy = False if error is not None: self._session = None self._set_status(f"录制会话启动失败: {error}") self._refresh_button_states() return self._auto_loop_active = True self._rec_paused = False self._update_flow_label() self._refresh_button_states() if rec_mode == _MODE_CHECKPOINT: self._auto_loop_active = False self._set_status("入口状态已采集；可手动添加界面检查点。") else: self.target_var.set("动作录制中：等待用户操作") self._set_status("入口状态已采集，动作录制已就绪。") self._schedule_heartbeat() def _start_auto_loop(self, *, first_delay_ms: int = _AUTO_START_DELAY_MS) -> None: """Legacy interval loop OR action-driven recording start.""" self._pending_end = False if self._is_legacy_interval(): self._auto_loop_active = True self._update_flow_label() self._refresh_button_states() self._schedule_auto_next(first_delay_ms) return # Action-driven: the desktop baseline is intentionally lightweight; # full provider evidence is collected asynchronously after actions. entry_dump = ( self._desktop_entry_dump if self._current_mode() == target.SURFACE_WINDOWS else None ) self._start_recording_session(entry_dump=entry_dump) def _set_inspector_topmost(self, enabled: bool) -> None: try: self.root.attributes("-topmost", bool(enabled)) except Exception: pass def _ensure_inspector_visible(self) -> None: """Keep panel on-screen for progress; do not force-focus (avoids stealing FG).""" try: if str(self.root.state()) == "iconic": self.root.deiconify() except Exception: pass def _arm_auto_with_countdown(self, *, seconds: int = _START_COUNTDOWN_SEC) -> None: """Visible countdown, then enter mode-specific auto loop.""" if not self._flow_is_open(): return # Stop prior loop/timers without clearing the countdown we are about to arm. self._auto_loop_active = False self._cancel_auto_timer() self._cancel_countdown() self._ensure_inspector_visible() # Stay topmost for progress; capture still uses focus + exclude self. self._set_inspector_topmost(True) self._countdown_left = max(1, int(seconds)) self._update_flow_label() mode = self._current_mode() if mode == target.SURFACE_BROWSER: url = (self.start_url_var.get() or "").strip() behavior = ( "Legacy：打开浏览器后每 2 秒采集一次" if self._is_legacy_interval() else "打开浏览器后监听用户动作，并采集动作后的稳定状态" ) self._set_status( f"倒计时 {self._countdown_left} 秒：将打开浏览器并进入 {url or '(起始 URL)'}" ) self._set_detail( f"流程：{self.flow_dir}\n\n" f"- 模式：浏览器自动化\n" f"- 起始 URL：{url or '(空，自行在浏览器中打开目标站)'}\n" f"- 倒计时结束后：{behavior}\n" f"- 可点「结束流程」或取消「自动拾取」中止\n" ) else: behavior = ( "Legacy：识别窗口后每 2 秒采集一次" if self._is_legacy_interval() else "识别窗口后监听用户动作，并采集动作后的稳定状态" ) self._set_status( f"倒计时 {self._countdown_left} 秒：请把要采集的桌面窗口点到最前端" "（本面板置顶看进度；不强制放大目标窗）。" ) self._set_detail( f"流程：{self.flow_dir}\n\n" f"- 模式：桌面自动化\n" f"- {self._countdown_left} 秒内请点击目标窗口（给它焦点）\n" f"- 倒计时结束后：{behavior}\n" f"- 可点「结束流程」或取消「自动拾取」中止\n" ) self._refresh_button_states() self._countdown_after_id = self.root.after(1000, self._countdown_tick) def _countdown_tick(self) -> None: self._countdown_after_id = None if not self._flow_is_open(): self._countdown_left = 0 self._refresh_button_states() return if not self.auto_pick_var.get(): self._countdown_left = 0 self._refresh_button_states() self._set_status("已取消自动拾取倒计时；可手动点「拾取当前页」。") return self._countdown_left -= 1 if self._countdown_left > 0: mode = self._current_mode() if mode == target.SURFACE_BROWSER: self._set_status( f"倒计时 {self._countdown_left} 秒：准备打开浏览器…" ) else: self._set_status( f"倒计时 {self._countdown_left} 秒：请把要采集的窗口点到最前端…" ) self._refresh_button_states() self._countdown_after_id = self.root.after(1000, self._countdown_tick) return self._countdown_left = 0 self._refresh_button_states() if self._current_mode() == target.SURFACE_BROWSER: self._set_status("倒计时结束，正在打开浏览器…") self._begin_auto_browser() else: self._set_status("倒计时结束，正在识别最前端窗口…") self._begin_auto_with_probe() def _begin_auto_browser(self) -> None: """Launch Playwright with start URL, then enter auto loop (no Win32 probe).""" if self._busy: return self._busy = True self._auto_loop_active = False self._refresh_button_states() self._set_inspector_topmost(True) self._ensure_inspector_visible() url = (self.start_url_var.get() or "").strip() status_url = url or "(未填起始 URL，打开空白页后可自行导航)" self._set_status(f"浏览器模式：正在打开 {status_url} …") self._set_detail( f"启动 Playwright…\n\n" f"- 起始 URL：{url or '(空)'}\n" f"- profile：{kit_browser.profile_dir()}\n" f"- 打开完成后进入{'Legacy 间隔采集' if self._is_legacy_interval() else '动作录制'}\n" ) def work() -> None: try: # 有 URL 才 force_goto；空则只启浏览器 sess = kit_browser.ensure_session( start_url=url or "about:blank", force_goto=bool(url), ) self._post_ui(self._on_browser_ready, sess, url) except Exception as exc: self._post_ui(self._on_frontmost_fail, exc) threading.Thread(target=work, daemon=True).start() def _on_browser_ready(self, sess: dict, url: str) -> None: self._busy = False self._refresh_button_states() live = str(sess.get("url") or "about:blank") if url and (not live or live == "about:blank" or live.startswith("about:")): self._on_frontmost_fail( RuntimeError( f"ERROR:NAVIGATION_FAILED requested={url!r} actual={live!r}" ) ) return navigation = dict(sess.get("navigation") or {}) nav_warning = str(navigation.get("warning") or "") self._set_status( f"浏览器已就绪：{live} — " + ("开始 Legacy 间隔采集。" if self._is_legacy_interval() else "正在采集入口状态。") ) self._set_detail( f"采集目标（Playwright）:\n" f" mode=browser\n" f" start_url={url or '(空)'}\n" f" live_url={live}\n" f" profile={sess.get('profile_dir')}\n" f" reused={sess.get('reused')}\n\n" + (f"导航警告：{nav_warning}\n\n" if nav_warning else "") + ( "即将进入 Legacy 自动循环（步间 2 秒）…\n" if self._is_legacy_interval() else "即将启动动作录制器…\n" ) + ( "" if url else "未填起始 URL：请在弹出的浏览器中打开目标站（空页会自动跳过写入）。\n" ) ) self._set_inspector_topmost(True) self._ensure_inspector_visible() self._start_auto_loop(first_delay_ms=_AUTO_START_DELAY_MS) def _begin_auto_with_probe(self) -> None: """Desktop mode: resolve frontmost target, warmup best-effort, then loop.""" if self._busy: return self._busy = True self._auto_loop_active = False self._refresh_button_states() # Keep topmost for progress; target = focused window (Inspector hwnd excluded). self._set_inspector_topmost(True) self._ensure_inspector_visible() self._set_status( "桌面模式：正在识别最前端窗口（焦点窗=采集目标，已排除本面板）…" ) self._set_detail( "识别最前端窗口中…\n\n" "- 模式：桌面自动化（强制 UIA）\n" "- 规则：谁有焦点就采谁（与窗口大小无关）\n" "- 若是企微：自动调用 kit ensure_ui_ready\n" ) exclude_hwnds = self._exclude_hwnds() def work() -> None: try: uia.ensure_com_sta() report = uia.prepare_frontmost_for_capture( exclude_hwnds=exclude_hwnds, timeout_sec=_FRONTMOST_WAIT_SEC, ) self._post_ui(self._on_frontmost_ready, report) except Exception as exc: self._post_ui(self._on_frontmost_fail, exc) threading.Thread(target=work, daemon=True).start() def _on_frontmost_ready(self, report: dict) -> None: self._busy = False self._refresh_button_states() win = report.get("window") or {} if win.get("pid"): self.sticky_pid = int(win["pid"]) bounds = dict(win.get("rect") or win.get("bounds") or {}) root_row = { "name": str(win.get("title") or ""), "value": "", "automation_id": "", "control_type": "WindowControl", "class_name": str(win.get("class_name") or ""), "framework_id": "Win32", "is_enabled": True, "is_offscreen": False, "bounds": bounds, "window_scope": dict(win), } self._desktop_entry_dump = DumpResult( surface=target.SURFACE_WINDOWS, driver="uiautomation-light", window=dict(win), tree={**root_row, "children": []}, index=[dict(root_row)], locatable=[dict(root_row)], stats={ "capture_kind": "entry_light", "nodes_kept": 1, "locatable_count": 1, "provider_enrichment": "deferred", }, selection={ "strategy": str(report.get("strategy") or "foreground_only"), "entry_light": True, }, use_hwnd_screenshot=False, ) import kit_wake kit = report.get("kit_wake") or {} kit_line = kit_wake.format_kit_wake_line(kit) sparse_note = "" if report.get("provider_cold") or report.get("uia_sparse"): if kit.get("reason") == "kit_not_installed": sparse_note = ( f"（仍偏稀 FindAll={report.get('findall')}：未安装 platform-kit，" "企微无法走 n2-wake）" ) elif kit.get("applicable") and not kit.get("ok"): sparse_note = ( f"（kit 暖树未成功 FindAll={report.get('findall')} " f"code={kit.get('error_code')}；仍按此窗采集）" ) else: sparse_note = ( f"（UIA 仍偏稀 FindAll={report.get('findall')}；仍按此前端窗采集）" ) self._set_status( f"已锁定桌面窗：{win.get('title')!r} " f"findall={report.get('findall')} — " + ("开始 Legacy 间隔采集。" if self._is_legacy_interval() else "正在采集入口状态。") + sparse_note ) self._set_detail( f"采集目标（桌面 UIA）:\n" f" mode=windows\n" f" title={win.get('title')!r}\n" f" class={win.get('class_name')!r}\n" f" hwnd={win.get('hwnd')} pid={win.get('pid')}\n" f" findall={report.get('findall')} sparse={report.get('uia_sparse')} " f"cold={report.get('provider_cold')}\n" f" {kit_line}\n\n" + ( "即将进入 Legacy 自动循环（步间 2 秒）…\n" if self._is_legacy_interval() else "即将启动动作录制器…\n" ) ) self._set_inspector_topmost(True) self._ensure_inspector_visible() self._start_auto_loop(first_delay_ms=_AUTO_START_DELAY_MS) def _on_frontmost_fail(self, exc: BaseException) -> None: self._busy = False self._auto_loop_active = False # Keep auto checked (default on); user can retry or end flow. self._set_inspector_topmost(True) self._ensure_inspector_visible() self._refresh_button_states() self._set_status(f"未能锁定最前端目标: {exc}") messagebox.showerror("UI Inspector", f"未能锁定最前端目标:\n{exc}") def _schedule_auto_next(self, delay_ms: int) -> None: self._cancel_auto_timer() if not self._auto_loop_active or not self._flow_is_open(): return self._auto_after_id = self.root.after(delay_ms, self._auto_loop_tick) def _auto_loop_tick(self) -> None: self._auto_after_id = None if not self._auto_loop_active or not self._flow_is_open(): return if self._busy: # Should not happen in serial design; re-check soon. self._schedule_auto_next(200) return self.on_pick_page(auto=True) def _continue_auto_loop_after_pick(self) -> None: if self._pending_end: self._pending_end = False self._stop_auto_loop() self._finish_end_flow() return if self._auto_loop_active and self._flow_is_open(): self._set_status( f"自动循环：第 {self.flow.get('step_count') if self.flow else '?'} 步已处理；" f"{_AUTO_GAP_MS // 1000} 秒后开始下一拍。点「结束流程」退出循环。" ) self._schedule_auto_next(_AUTO_GAP_MS) def on_auto_toggle(self) -> None: if self.auto_pick_var.get(): if ( self._flow_is_open() and not self._auto_loop_active and not self._busy and not self._in_start_countdown() ): self._arm_auto_with_countdown() else: self._update_flow_label() else: was = self._auto_loop_active or self._in_start_countdown() self._stop_auto_loop() self._update_flow_label() self._refresh_button_states() if was and self._flow_is_open(): self._set_status("已停止自动录制/倒计时；仍可手动点「采集界面检查点」。") elif self._flow_is_open(): self._set_status("自动录制已关闭；可手动点「采集界面检查点」。") def on_open_out(self) -> None: path = self.flow_dir if self.flow_dir else self.captures_root path.mkdir(parents=True, exist_ok=True) try: os.startfile(str(path)) # type: ignore[attr-defined] except Exception: subprocess.Popen(["explorer", str(path)]) def on_start_flow(self) -> None: if self._busy: return mode = self._current_mode() if mode == target.SURFACE_ANDROID: messagebox.showwarning( "UI Inspector", "手机自动化尚未实现。\n请先选择「浏览器自动化」或「桌面自动化」。", ) return self._persist_prefs() if self.flow and self.flow.get("status") == "open": if not messagebox.askyesno( "UI Inspector", "当前已有进行中的流程。\n是否结束旧流程并开始新流程？", ): return self._stop_auto_loop() self._pending_end = False try: art.end_flow(self.flow_dir, self.flow) # type: ignore[arg-type] except Exception: pass self._stop_session() name = (self.flow_name_var.get() or "").strip() if not name: default_name = _flow_name_for_mode(mode) name = simpledialog.askstring( "流程名", "请输入流程名称：", initialvalue=default_name ) if not name: return self.flow_name_var.set(name) try: self.flow_dir = art.create_flow( self.captures_root, flow_name=name, recording_mode=self._current_recording_mode(), ) self.flow = art.load_flow(self.flow_dir) self._pending_end = False self._warmed_pids.clear() self._current_live_action = None self._desktop_entry_dump = None self._stop_session() self._update_flow_label() self._refresh_button_states() if self._current_recording_mode() == _MODE_CHECKPOINT: self.auto_pick_var.set(False) if self.auto_pick_var.get() and self._current_recording_mode() != _MODE_CHECKPOINT: # No modal / no minimize: keep panel visible for countdown + progress. self._arm_auto_with_countdown(seconds=_START_COUNTDOWN_SEC) else: self._stop_auto_loop() self._ensure_inspector_visible() if mode == target.SURFACE_BROWSER: url = (self.start_url_var.get() or "").strip() self._set_status( f"已开始流程：{self.flow_dir.name}。" + ( f"将打开 {url}；请点「采集界面检查点」。" if url else "已启浏览器（无起始 URL）；请自行打开目标站后点「采集界面检查点」。" ) ) # Eager open browser so manual pick is ready def _open() -> None: try: kit_browser.ensure_session( start_url=url or "about:blank", force_goto=bool(url), ) except Exception: pass threading.Thread(target=_open, daemon=True).start() else: self._set_status( f"已开始流程：{self.flow_dir.name}。请翻到目标页后点「采集界面检查点」。" ) # Manual mode owns no surface recorder; it only keeps the v2 # manager available for explicit checkpoints. self._start_recording_session(checkpoint_only=True) self._set_detail( f"流程已创建：{self.flow_dir}\n\n" f"- 模式：{mode}\n" f"- 录制：{self._current_recording_mode()}\n" f"- flow.json / action_flow.json / element_catalog.json / steps/\n" f"- 手动点「采集界面检查点」；或勾选自动录制用户操作。\n" ) except Exception as exc: messagebox.showerror("UI Inspector", str(exc)) def _finish_end_flow(self) -> None: if not self.flow_dir or not self.flow: return if self.flow.get("status") != "open": self._refresh_button_states() return session = self._session flow_dir = self.flow_dir flow = self.flow self._session = None self._busy = True self._refresh_button_states() self._set_status("正在后台刷新动作队列并结束流程……") def work() -> None: try: if session is not None: session.stop() latest_flow = art.load_flow(flow_dir) ended = art.end_flow(flow_dir, latest_flow) self._post_ui(self._on_end_flow_finished, ended, None) except Exception as exc: self._post_ui(self._on_end_flow_finished, None, exc) threading.Thread( target=work, name="ui-inspector-session-stop", daemon=True ).start() def _on_end_flow_finished( self, ended_flow: dict | None, error: BaseException | None ) -> None: self._busy = False if error is not None or ended_flow is None: self._refresh_button_states() messagebox.showerror("UI Inspector", str(error or "结束流程失败")) return try: self.flow = ended_flow self._update_flow_label() self._refresh_button_states() n = self.flow.get("step_count") or 0 v2 = self.flow.get("v2") or {} self._set_status( f"流程已结束：actions={v2.get('action_count', '?')}，" f"states={v2.get('state_count', '?')}，" f"observations={v2.get('observation_count', '?')}，" f"legacy_steps={n}。" "可再点「开始流程」开新文档。" ) self._set_detail( f"流程已结束：{self.flow_dir}\n\n" f"- v1 steps: {n}\n" f"- v2 actions: {v2.get('action_count')}\n" f"- 请打开 SUMMARY.md / action_flow.json / element_catalog.json\n" ) messagebox.showinfo( "UI Inspector", f"流程已结束\nactions={v2.get('action_count')} " f"states={v2.get('state_count')} " f"observations={v2.get('observation_count')} " f"legacy_steps={n}\n{self.flow_dir}", ) except Exception as exc: messagebox.showerror("UI Inspector", str(exc)) def on_pause_resume(self) -> None: if not self._session or not self._flow_is_open(): return if self._rec_paused: self._session.resume() self._rec_paused = False self.pause_btn.configure(text="暂停") self._set_status("已继续录制。") else: self._session.pause() self._rec_paused = True self.pause_btn.configure(text="继续") self._set_status("已暂停录制（Inspector 自身操作不会写入）。") self._update_rec_stats() def on_end_flow(self) -> None: if not self.flow_dir or not self.flow: messagebox.showwarning("UI Inspector", "没有进行中的流程。") return if self.flow.get("status") != "open": messagebox.showinfo("UI Inspector", "流程已经结束。") return self._stop_auto_loop() # Keep「自动录制」勾选状态（默认开），方便下一轮直接开始。 if self._busy: self._pending_end = True self._set_status("已请求结束：等待当前这一拍 dump/保存完成…") self._refresh_button_states() return self._pending_end = False self._finish_end_flow() def on_pick_page(self, *, auto: bool = False) -> None: """Manual checkpoint or legacy auto dump. Action-driven mode uses Recorder.""" if self._busy: return if not self._flow_is_open(): if not auto: messagebox.showwarning("UI Inspector", "请先点「开始流程」。") return if auto and not self._auto_loop_active: return # In action-driven mode, auto ticks should not dump steps. if auto and not self._is_legacy_interval(): return self._busy = True self._refresh_button_states() self._show_picking_placeholder(auto=auto) self.root.update_idletasks() # Capture Tk-owned configuration before entering the worker. mode = self._current_mode() under_cursor = bool(self.under_cursor_var.get()) prefer_search = bool(self.prefer_wecom_search_var.get()) start_url = (self.start_url_var.get() or "").strip() sticky_pid = self.sticky_pid exclude_hwnds = self._exclude_hwnds() flow_dir = self.flow_dir synthetic_browser_info = ( self._synthetic_browser_info() if mode == target.SURFACE_BROWSER else None ) def work() -> None: try: uia.ensure_com_sta() flow = art.load_flow(flow_dir) # type: ignore[arg-type] if flow.get("status") != "open": raise RuntimeError("流程已结束，取消本步拾取。") if mode == target.SURFACE_ANDROID: raise RuntimeError( "ERROR:SURFACE_UNSUPPORTED 手机自动化尚未实现。" ) surface = mode sel_meta: dict = { "pick_mode": mode, "sticky_pid": sticky_pid, } if surface == target.SURFACE_BROWSER: kit_browser.ensure_session( start_url=start_url or "about:blank", force_goto=False, ) info = synthetic_browser_info # Best-effort Win32 chrome window for meta only try: win_info, cands, win_meta = uia.resolve_capture_target( exclude_hwnds=exclude_hwnds, prefer_pid=None, under_cursor=False, prefer_wecom_search_popup=False, ) if target.surface_from_window_info(win_info) == ( target.SURFACE_BROWSER ): info = win_info sel_meta.update(win_meta) sel_meta["candidate_count"] = len(cands) except Exception: sel_meta["strategy"] = "playwright_session" sel_meta["resolved_surface"] = surface sel_meta["start_url"] = start_url else: info, cands, win_meta = uia.resolve_capture_target( exclude_hwnds=exclude_hwnds, prefer_pid=None if auto else sticky_pid, under_cursor=under_cursor, prefer_wecom_search_popup=prefer_search, ) sel_meta.update(win_meta) sel_meta["candidate_count"] = len(cands) sel_meta["resolved_surface"] = surface driver = get_driver(surface) manual_warm_key = ( (int(info.pid), str(info.class_name or "")) if surface == target.SURFACE_WINDOWS else int(info.pid or 0) ) warmup_mode = ( "light" if manual_warm_key in self._warmed_pids else "full" ) result = driver.dump( info=info, selection_meta=sel_meta, max_depth=self.max_depth, max_nodes=self.max_nodes, warmup_mode=warmup_mode, warmed_pids=self._warmed_pids, ) if info.pid: self._warmed_pids.add(manual_warm_key) skip_reason = skip_policy.should_skip_empty_dump( surface=result.surface, url=result.url, locatable=result.locatable, stats=result.stats, ) if skip_reason: msg = skip_policy.skip_reason_text( skip_reason, url=result.url ) self._post_ui( self._on_pick_skipped, msg, auto=auto, surface=result.surface, url=result.url, ) return # Dual-write v2 checkpoint when session exists (manual) if self._session is not None and not auto: try: mgr = self._session.manager before = mgr.snapshots.latest after = mgr.snapshots.capture_from_dump( result, change_reason="checkpoint", force_persist=True ) mgr._finalize_action( action_type="checkpoint", before=before, after=after, target={}, input_payload={}, raw_event={"meta": {"source": "manual_checkpoint"}}, confidence=0.9, ) except Exception: pass window = result.window locatable = result.locatable index = result.index stats = result.stats label = art.resolve_step_label( flow, window_title=window.get("title") or info.title or "", window_class=window.get("class_name") or info.class_name or "", locatable=locatable, ) step_dir, step_idx, step_name = art.add_step_dir( self.flow_dir, # type: ignore[arg-type] flow, window_title=window.get("title") or info.title or "", window_class=window.get("class_name") or info.class_name or "", locatable=locatable, label=label, ) meta = art.build_meta( window=window, scene=flow.get("flow_name") or "flow", surface=result.surface, driver=result.driver, url=result.url, extra={ "flow_dir": str(self.flow_dir), "step_index": step_idx, "step_name": step_name, "step_label": label, "auto_pick": auto, "dump_stats": stats, "selection": result.selection, }, ) art.write_json(step_dir / "meta.json", meta) art.write_json(step_dir / "tree.json", result.tree) art.write_jsonl(step_dir / "index.jsonl", index) art.write_json(step_dir / "locatable.json", locatable) png_ok = False if result.screenshot_bytes: try: (step_dir / "screen.png").write_bytes(result.screenshot_bytes) png_ok = True except Exception: png_ok = False elif result.use_hwnd_screenshot: png_ok = art.capture_window_png(info.hwnd, step_dir / "screen.png") noteworthy = art.noteworthy_from_index(index) art.write_step_summary( step_dir, meta=meta, stats=stats, noteworthy=noteworthy ) art.register_step( self.flow_dir, # type: ignore[arg-type] flow, step_index=step_idx, step_name=step_name, window=window, stats=stats, label=label, surface=result.surface, url=result.url, ) flow = art.load_flow(self.flow_dir) # type: ignore[arg-type] import kit_wake warm = stats.get("warmup") or {} kit = stats.get("kit_wake") or {} kit_line = kit_wake.format_kit_wake_line(kit) sparse_note = "" if stats.get("uia_sparse"): sparse_note = "\n⚠ sparse=true：可定位候选偏少，请结合 screen.png。\n" mode = "自动" if auto else "手动" url_line = f"url: {result.url}\n" if result.url else "" header = ( f"流程步骤: #{step_idx} `{step_name}`（{mode}）\n" f"surface: {result.surface} ({target.surface_label(result.surface)}) " f"driver={result.driver}\n" f"{url_line}" f"窗口: title={window.get('title')!r} class={window.get('class_name')!r} " f"pid={window.get('pid')} hwnd={window.get('hwnd')}\n" f"模式: {mode} 选窗: {sel_meta.get('strategy')}\n" f"kit: {kit_line}\n" f"软poke: findall {warm.get('findall_before')} → {warm.get('findall_after')} " f"woke={warm.get('woke')}\n" f"统计: kept={stats.get('nodes_kept')} findall={stats.get('findall_count')} " f"locatable={stats.get('locatable_count')} png={png_ok} " f"sparse={stats.get('uia_sparse')} cold={stats.get('provider_cold')} " f"merge={stats.get('merge_same_pid_allowed')}\n" f"已写入: {step_dir}\n" f"流程文档: {self.flow_dir}\n" f"{sparse_note}" f"{'=' * 60}\n" ) body = ( header + "\n" + (result.locatable_text or "") + "\n\n" + ("=" * 60) + "\n控件树:\n" + (result.tree_text or "") ) self._post_ui( self._on_pick_ok, window, flow, body, step_name, auto=auto, surface=result.surface, ) except Exception as exc: self._post_ui(self._on_fail, exc, auto=auto) threading.Thread(target=work, daemon=True).start() def _on_pick_ok( self, window: dict, flow: dict, text: str, step_name: str, *, auto: bool = False, surface: str = "windows", ) -> None: self._busy = False self.flow = flow self.last_window = dict(window or {}) if window.get("pid"): try: self.sticky_pid = int(window["pid"]) except Exception: pass self._update_flow_label() self._refresh_button_states() url = window.get("url") or "" url_bit = f" url={url}" if url else "" self.target_var.set( f"本步: surface={surface} title={window.get('title')!r} " f"class={window.get('class_name')!r}{url_bit} | step=`{step_name}`" ) self._set_detail(text) if not auto: self._set_status( f"第 {flow.get('step_count')} 步已写入。可换页继续拾取，或点「结束流程」。" ) try: self.root.lift() self.root.attributes("-topmost", True) except Exception: pass if self._pending_end: self._pending_end = False self._finish_end_flow() return # auto: either pending end, or continue loop after gap self._continue_auto_loop_after_pick() def _on_pick_skipped( self, message: str, *, auto: bool = False, surface: str = "browser", url: str = "", ) -> None: """Blank/empty browser dump: do not write a step; keep looping if auto.""" self._busy = False self._refresh_button_states() self._update_flow_label() url_bit = f" url={url}" if url else "" self.target_var.set(f"已跳过空页（{surface}）{url_bit}") n = int((self.flow or {}).get("step_count") or 0) detail = ( f"{message}\n\n" f"- 未写入 steps/（避免 about:blank / 空壳污染流程文档）\n" f"- 当前流程已有步骤数: {n}\n" f"- 打开目标站或等待页面加载后再拾取\n" ) if auto: detail += f"- 自动循环将在 {_AUTO_GAP_MS // 1000} 秒后继续\n" self._set_detail(detail) self._set_status(message) if not auto: try: self.root.lift() self.root.attributes("-topmost", True) except Exception: pass if self._pending_end: self._pending_end = False self._finish_end_flow() return self._continue_auto_loop_after_pick() def _on_fail(self, exc: BaseException, *, auto: bool = False) -> None: self._busy = False self._refresh_button_states() self._set_status(f"失败: {exc}") if not auto: messagebox.showerror("UI Inspector", str(exc)) if self._pending_end: self._pending_end = False self._finish_end_flow() return # Auto: do not pop modal; keep looping after gap (unless ending) self._continue_auto_loop_after_pick() def _on_close(self) -> None: if self._closed: return self._closed = True self._stop_auto_loop() self._pending_end = False self._stop_session() if self.flow_dir and self.flow and self.flow.get("status") == "open": try: latest_flow = art.load_flow(self.flow_dir) self.flow = art.end_flow(self.flow_dir, latest_flow) except Exception: pass try: self._persist_prefs() except Exception: pass try: kit_browser.shutdown() except Exception: pass try: self.root.destroy() except Exception: pass def run(self) -> None: self.captures_root.mkdir(parents=True, exist_ok=True) try: self.root.mainloop() except KeyboardInterrupt: pass finally: self._on_close() def parse_args(argv: list[str] | None = None) -> argparse.Namespace: p = argparse.ArgumentParser( description=( f"OpenClaw UI Inspector v{__version__}" "（browser/windows：动作录制、检查点与 Legacy 间隔采集）" ) ) p.add_argument("--out", type=Path, default=None, help="输出根目录") p.add_argument( "--scene", default="", help="默认流程名（空则按拾取模式：browser / desktop / mobile）", ) p.add_argument("--max-depth", type=int, default=16) p.add_argument("--max-nodes", type=int, default=8000) return p.parse_args(argv) def main(argv: list[str] | None = None) -> int: args = parse_args(argv) out = args.out.resolve() if args.out else art.default_captures_root(_HERE) try: uia.ensure_uia() except Exception as exc: print(f"ERROR: {exc}", file=sys.stderr) return 1 app = InspectorApp( captures_root=out, scene=args.scene, max_depth=args.max_depth, max_nodes=args.max_nodes, ) app.run() return 0 if __name__ == "__main__": raise SystemExit(main())