chendelian
633ecff4e2
Keep the last in-template copy on main so history can restore it; product continues in jiangchang-ui-recorder. Co-authored-by: Cursor <cursoragent@cursor.com>
2002 lines
80 KiB
Python
2002 lines
80 KiB
Python
"""Tk panel v0.14: Action-driven UI Automation Recorder (compatible with v1 dumps).
|
||
|
||
拾取模式（单选，记忆上次）：浏览器 / 桌面 / 手机。
|
||
- 浏览器：强制 Playwright；起始 URL 自动打开；Browser Recorder
|
||
- 桌面：强制 UIA；Windows Recorder（进程外轮询，无注入）
|
||
- 手机：占位未实现
|
||
|
||
录制模式：自动录制（默认）/ 引导录制 / 手动检查点；legacy 间隔 dump 可选。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import ctypes
|
||
import os
|
||
import queue
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
from ctypes import wintypes
|
||
from pathlib import Path
|
||
import tkinter as tk
|
||
from tkinter import messagebox, simpledialog, ttk
|
||
|
||
_HERE = Path(__file__).resolve().parent
|
||
if str(_HERE) not in sys.path:
|
||
sys.path.insert(0, str(_HERE))
|
||
|
||
import artifacts as art # noqa: E402
|
||
import desktop_uia as uia # noqa: E402
|
||
import kit_browser # noqa: E402
|
||
import skip_policy # noqa: E402
|
||
import target # noqa: E402
|
||
import user_prefs # noqa: E402
|
||
from surfaces.registry import get_driver # noqa: E402
|
||
from flow_session import FlowSession # noqa: E402
|
||
from drivers.base import DumpResult # noqa: E402
|
||
|
||
try:
|
||
from . import __version__
|
||
except Exception:
|
||
try:
|
||
import importlib.util
|
||
|
||
_spec = importlib.util.spec_from_file_location(
|
||
"ui_inspector_pkg", _HERE / "__init__.py"
|
||
)
|
||
_mod = importlib.util.module_from_spec(_spec) # type: ignore[arg-type]
|
||
assert _spec and _spec.loader
|
||
_spec.loader.exec_module(_mod)
|
||
__version__ = getattr(_mod, "__version__", "0.16.0")
|
||
except Exception:
|
||
__version__ = "0.16.0"
|
||
|
||
# Legacy interval mode: after each dump finishes, wait this long, then dump again.
|
||
_AUTO_GAP_MS = 2000
|
||
# Heartbeat for action-driven mode (flush idle input; do NOT create steps).
|
||
_HEARTBEAT_MS = 800
|
||
# After frontmost target is resolved, short delay before first dump.
|
||
_AUTO_START_DELAY_MS = 400
|
||
# After「开始流程」with auto-pick: countdown so user can focus the target app.
|
||
_START_COUNTDOWN_SEC = 5
|
||
_FRONTMOST_WAIT_SEC = 15.0
|
||
_STABILITY_TIMEOUT_SEC = 3.0
|
||
|
||
# Recording modes
|
||
_MODE_AUTO = "auto"
|
||
_MODE_GUIDED = "guided"
|
||
_MODE_CHECKPOINT = "checkpoint"
|
||
_MODE_LEGACY = "legacy_interval"
|
||
|
||
_WINDOW_RIGHT_MARGIN = 16
|
||
_WINDOW_BOTTOM_MARGIN = 16
|
||
_WINDOW_MIN_WIDTH = 520
|
||
_WINDOW_MIN_HEIGHT = 600
|
||
|
||
# Default flow-name slugs aligned with the pick-mode radio labels.
|
||
# Switching mode only auto-updates the field when it still holds one of these.
|
||
_MODE_FLOW_NAMES = {
|
||
target.SURFACE_BROWSER: "browser",
|
||
target.SURFACE_WINDOWS: "desktop",
|
||
target.SURFACE_ANDROID: "mobile",
|
||
}
|
||
_DEFAULT_FLOW_NAME_SLUGS = frozenset(_MODE_FLOW_NAMES.values())
|
||
|
||
|
||
def _flow_name_for_mode(mode: str) -> str:
|
||
return _MODE_FLOW_NAMES.get(mode, "desktop")
|
||
|
||
|
||
def _clamp(value: int, low: int, high: int) -> int:
|
||
return max(low, min(high, value))
|
||
|
||
|
||
def _default_window_geometry(
|
||
work_area: tuple[int, int, int, int],
|
||
*,
|
||
side: str = "right",
|
||
) -> tuple[int, int, int, int]:
|
||
"""Return a narrow console geometry inside a monitor work area.
|
||
|
||
The top gap deliberately leaves the target application's title-bar controls
|
||
unobstructed. Coordinates may be negative for monitors left of the primary.
|
||
"""
|
||
|
||
left, top, right, bottom = work_area
|
||
work_width = max(1, right - left)
|
||
work_height = max(1, bottom - top)
|
||
width = _clamp(round(work_width * 0.30), 560, 620)
|
||
width = min(width, max(420, work_width - 32))
|
||
top_gap = _clamp(round(work_height * 0.07), 64, 88)
|
||
height = max(480, work_height - top_gap - _WINDOW_BOTTOM_MARGIN)
|
||
x = (
|
||
left + _WINDOW_RIGHT_MARGIN
|
||
if side == "left"
|
||
else right - width - _WINDOW_RIGHT_MARGIN
|
||
)
|
||
y = top + top_gap
|
||
return width, height, x, y
|
||
|
||
|
||
def _format_geometry(width: int, height: int, x: int, y: int) -> str:
|
||
x_part = f"+{x}" if x >= 0 else str(x)
|
||
y_part = f"+{y}" if y >= 0 else str(y)
|
||
return f"{width}x{height}{x_part}{y_part}"
|
||
|
||
|
||
def _format_live_action(item: dict) -> str:
|
||
"""Compact, surface-neutral live action text for the result panel."""
|
||
labels = {
|
||
"click": "点击", "fill": "输入", "type": "输入",
|
||
"press_key": "按键", "navigation": "导航", "checkpoint": "检查点",
|
||
}
|
||
phases = {
|
||
"captured": "已捕获，证据采集中",
|
||
"evidence": "证据已更新",
|
||
"evidence_failed": "证据采集失败",
|
||
}
|
||
action_type = str(item.get("action_type") or "unknown")
|
||
lines = [
|
||
f"#{item.get('sequence')} {labels.get(action_type, action_type)} · "
|
||
f"{phases.get(str(item.get('phase') or ''), str(item.get('phase') or ''))}",
|
||
f" 表面: {item.get('surface') or '-'} 目标: "
|
||
f"{item.get('target_name') or '-'} ({item.get('target_type') or '-'})",
|
||
]
|
||
if item.get("input_summary"):
|
||
lines.append(f" 输入: {item['input_summary']}")
|
||
if item.get("window_class"):
|
||
lines.append(
|
||
f" 窗口: {item.get('window_title') or '-'} [{item.get('window_class')}]"
|
||
)
|
||
if item.get("locator"):
|
||
lines.append(
|
||
f" 定位: {item['locator']} match={item.get('match_count')} "
|
||
f"unique={item.get('unique')}"
|
||
)
|
||
elif item.get("phase") == "evidence_failed":
|
||
lines.append(f" 错误: {item.get('error') or '未知错误'}")
|
||
else:
|
||
lines.append(" 定位: 等待异步证据")
|
||
return "\n".join(lines)
|
||
|
||
|
||
def _merge_current_live_action(current: dict | None, incoming: dict) -> dict | None:
|
||
"""Keep the newest action while allowing its evidence phase to replace it."""
|
||
if not str(incoming.get("id") or ""):
|
||
return current
|
||
if current is None:
|
||
return dict(incoming)
|
||
current_sequence = int(current.get("sequence") or 0)
|
||
incoming_sequence = int(incoming.get("sequence") or 0)
|
||
if incoming_sequence < current_sequence:
|
||
return current
|
||
return dict(incoming)
|
||
|
||
|
||
class InspectorApp:
|
||
def __init__(
|
||
self,
|
||
*,
|
||
captures_root: Path,
|
||
scene: str = "",
|
||
max_depth: int = 16,
|
||
max_nodes: int = 8000,
|
||
) -> None:
|
||
self.captures_root = captures_root
|
||
self.scene = scene
|
||
self.max_depth = max_depth
|
||
self.max_nodes = max_nodes
|
||
|
||
self.flow_dir: Path | None = None
|
||
self.flow: dict | None = None
|
||
self.last_window: dict | None = None
|
||
self.sticky_pid: int | None = None
|
||
# Browser entries use pid; native entries use (pid, window class) so a
|
||
# WeCom SearchResultWindow2 is not mistaken for its already-warmed main
|
||
# WeWorkWindow merely because both belong to the same process.
|
||
self._warmed_pids: set[object] = set()
|
||
self._busy = False
|
||
self._auto_loop_active = False
|
||
self._pending_end = False
|
||
self._auto_after_id: str | None = None
|
||
self._countdown_after_id: str | None = None
|
||
self._countdown_left = 0
|
||
self._session: FlowSession | None = None
|
||
self._desktop_entry_dump: DumpResult | None = None
|
||
self._heartbeat_after_id: str | None = None
|
||
self._rec_paused = False
|
||
self._current_live_action: dict | None = None
|
||
self._ui_queue: queue.Queue[tuple[object, tuple, dict]] = queue.Queue()
|
||
self._ui_poll_after_id: str | None = None
|
||
self._closed = False
|
||
|
||
prefs = user_prefs.load_prefs()
|
||
initial_mode = str(prefs.get("mode") or user_prefs.DEFAULT_MODE)
|
||
if initial_mode in {"mobile", "phone", "ios"}:
|
||
initial_mode = target.SURFACE_ANDROID
|
||
if initial_mode not in {
|
||
target.SURFACE_BROWSER,
|
||
target.SURFACE_WINDOWS,
|
||
target.SURFACE_ANDROID,
|
||
}:
|
||
initial_mode = user_prefs.DEFAULT_MODE
|
||
scene_name = (scene or "").strip()
|
||
initial_flow_name = scene_name or _flow_name_for_mode(initial_mode)
|
||
|
||
self.root = tk.Tk()
|
||
self.root.title(f"OpenClaw UI Inspector v{__version__}")
|
||
self._dock_side = "right"
|
||
self._apply_default_geometry()
|
||
self.root.attributes("-topmost", True)
|
||
|
||
self.flow_name_var = tk.StringVar(value=initial_flow_name)
|
||
self.mode_var = tk.StringVar(value=initial_mode)
|
||
self.start_url_var = tk.StringVar(value=str(prefs.get("start_url") or ""))
|
||
# Standard UI always records actions automatically. Legacy strategies
|
||
# remain supported internally for old captures and developer tooling.
|
||
self.auto_pick_var = tk.BooleanVar(value=True)
|
||
self.recording_mode_var = tk.StringVar(value=_MODE_AUTO)
|
||
# 默认开：悬停即可采搜索浮层，不必点进会话；均可随时取消勾选回退旧行为
|
||
self.under_cursor_var = tk.BooleanVar(value=True)
|
||
self.prefer_wecom_search_var = tk.BooleanVar(value=True)
|
||
self.flow_var = tk.StringVar(value="流程：未开始（请先点「开始流程」）")
|
||
self.target_var = tk.StringVar(value="等待开始流程")
|
||
self.rec_stats_var = tk.StringVar(
|
||
value="recorder: idle | actions=0 states=0 observations=0 elements=0"
|
||
)
|
||
self.status_var = tk.StringVar(
|
||
value="选择模式后点击「开始流程」。"
|
||
)
|
||
|
||
self._build_ui()
|
||
for var in (
|
||
self.flow_var,
|
||
self.rec_stats_var,
|
||
self.target_var,
|
||
self.status_var,
|
||
):
|
||
var.trace_add("write", self._on_status_var_changed)
|
||
self._render_status_summary()
|
||
self._refresh_mode_widgets()
|
||
self.root.protocol("WM_DELETE_WINDOW", self._on_close)
|
||
self._schedule_ui_queue_poll()
|
||
|
||
def _current_monitor_work_area(self) -> tuple[int, int, int, int]:
|
||
"""Get the work area of the monitor containing the mouse pointer."""
|
||
|
||
if sys.platform == "win32":
|
||
try:
|
||
class MONITORINFO(ctypes.Structure):
|
||
_fields_ = [
|
||
("cbSize", wintypes.DWORD),
|
||
("rcMonitor", wintypes.RECT),
|
||
("rcWork", wintypes.RECT),
|
||
("dwFlags", wintypes.DWORD),
|
||
]
|
||
|
||
point = wintypes.POINT()
|
||
user32 = ctypes.windll.user32
|
||
user32.GetCursorPos.argtypes = [ctypes.POINTER(wintypes.POINT)]
|
||
user32.GetCursorPos.restype = wintypes.BOOL
|
||
user32.MonitorFromPoint.argtypes = [wintypes.POINT, wintypes.DWORD]
|
||
user32.MonitorFromPoint.restype = wintypes.HANDLE
|
||
user32.GetMonitorInfoW.argtypes = [
|
||
wintypes.HANDLE,
|
||
ctypes.POINTER(MONITORINFO),
|
||
]
|
||
user32.GetMonitorInfoW.restype = wintypes.BOOL
|
||
if user32.GetCursorPos(ctypes.byref(point)):
|
||
monitor = user32.MonitorFromPoint(point, 2)
|
||
info = MONITORINFO()
|
||
info.cbSize = ctypes.sizeof(info)
|
||
if monitor and user32.GetMonitorInfoW(monitor, ctypes.byref(info)):
|
||
area = info.rcWork
|
||
return area.left, area.top, area.right, area.bottom
|
||
except Exception:
|
||
pass
|
||
return 0, 0, self.root.winfo_screenwidth(), self.root.winfo_screenheight()
|
||
|
||
def _apply_default_geometry(self) -> None:
|
||
width, height, x, y = _default_window_geometry(
|
||
self._current_monitor_work_area(), side=self._dock_side
|
||
)
|
||
self.root.geometry(_format_geometry(width, height, x, y))
|
||
self.root.minsize(min(_WINDOW_MIN_WIDTH, width), min(_WINDOW_MIN_HEIGHT, height))
|
||
|
||
def _toggle_dock(self) -> None:
|
||
self._dock_side = "left" if self._dock_side == "right" else "right"
|
||
self._apply_default_geometry()
|
||
self.dock_btn.configure(text="靠右" if self._dock_side == "left" else "靠左")
|
||
|
||
def _on_status_var_changed(self, *_args: object) -> None:
|
||
self._render_status_summary()
|
||
|
||
def _render_status_summary(self) -> None:
|
||
if not hasattr(self, "status_text"):
|
||
return
|
||
if not self.flow:
|
||
headline = "● 未开始 · recorder idle"
|
||
else:
|
||
v2 = self.flow.get("v2") or {}
|
||
live = self._session.stats() if self._session else {}
|
||
status = str(self.flow.get("status") or "unknown")
|
||
status_name = {"open": "录制中", "ended": "已结束"}.get(status, status)
|
||
action_count = live.get("action_count", v2.get("action_count", 0))
|
||
state_count = live.get("state_count", v2.get("state_count", 0))
|
||
dropped = live.get("dropped_events", 0)
|
||
input_health = live.get("input_backend") or {}
|
||
input_suffix = ""
|
||
if input_health:
|
||
input_suffix = (
|
||
f" · input {input_health.get('raw_input_events', 0)}"
|
||
f" · {input_health.get('backend', 'unknown')}"
|
||
)
|
||
headline = (
|
||
f"● {status_name} · actions {action_count} · states {state_count}"
|
||
f" · dropped {dropped}{input_suffix}"
|
||
)
|
||
target_line = self.target_var.get().strip()
|
||
status_line = self.status_var.get().strip()
|
||
lines = [headline]
|
||
if target_line:
|
||
lines.append(target_line)
|
||
if status_line and status_line != target_line:
|
||
lines.append(status_line)
|
||
self.status_text.configure(state=tk.NORMAL)
|
||
self.status_text.delete("1.0", tk.END)
|
||
self.status_text.insert(tk.END, "\n".join(line for line in lines if line))
|
||
self.status_text.configure(state=tk.DISABLED)
|
||
|
||
def _post_ui(self, callback, *args, **kwargs) -> None:
|
||
"""Worker-safe UI handoff; workers must never call Tk directly."""
|
||
self._ui_queue.put((callback, args, kwargs))
|
||
|
||
def _schedule_ui_queue_poll(self) -> None:
|
||
self._ui_poll_after_id = self.root.after(40, self._drain_ui_queue)
|
||
|
||
def _drain_ui_queue(self) -> None:
|
||
self._ui_poll_after_id = None
|
||
for _ in range(100):
|
||
try:
|
||
callback, args, kwargs = self._ui_queue.get_nowait()
|
||
except queue.Empty:
|
||
break
|
||
try:
|
||
callback(*args, **kwargs) # type: ignore[operator]
|
||
except Exception as exc:
|
||
try:
|
||
self._set_status(f"后台结果处理失败: {exc}")
|
||
except Exception:
|
||
pass
|
||
try:
|
||
if self.root.winfo_exists():
|
||
self._schedule_ui_queue_poll()
|
||
except Exception:
|
||
pass
|
||
|
||
def _current_mode(self) -> str:
|
||
mode = (self.mode_var.get() or user_prefs.DEFAULT_MODE).strip().lower()
|
||
if mode in {"mobile", "phone", "ios"}:
|
||
return target.SURFACE_ANDROID
|
||
if mode not in {
|
||
target.SURFACE_BROWSER,
|
||
target.SURFACE_WINDOWS,
|
||
target.SURFACE_ANDROID,
|
||
}:
|
||
return user_prefs.DEFAULT_MODE
|
||
return mode
|
||
|
||
def _persist_prefs(self) -> None:
|
||
user_prefs.save_prefs(
|
||
mode=self._current_mode(),
|
||
start_url=(self.start_url_var.get() or "").strip(),
|
||
recording_mode=_MODE_AUTO,
|
||
)
|
||
|
||
def _current_recording_mode(self) -> str:
|
||
return _MODE_AUTO
|
||
|
||
def _is_legacy_interval(self) -> bool:
|
||
return self._current_recording_mode() == _MODE_LEGACY
|
||
|
||
def _on_mode_change(self) -> None:
|
||
self._persist_prefs()
|
||
self._sync_default_flow_name()
|
||
self._refresh_mode_widgets()
|
||
mode = self._current_mode()
|
||
if mode == target.SURFACE_BROWSER:
|
||
self._set_status(
|
||
"已选「浏览器自动化」：开始流程后将打开起始 URL 并 dump Playwright 页。"
|
||
)
|
||
elif mode == target.SURFACE_WINDOWS:
|
||
self._set_status(
|
||
"已选「桌面自动化」：开始流程后按光标/前台窗口做 UIA dump（不启浏览器）。"
|
||
)
|
||
else:
|
||
self._set_status("已选「手机自动化」：尚未实现，开始流程时会提示。")
|
||
|
||
def _sync_default_flow_name(self) -> None:
|
||
"""Keep flow name aligned with pick mode unless the user customized it."""
|
||
current = (self.flow_name_var.get() or "").strip()
|
||
if current and current not in _DEFAULT_FLOW_NAME_SLUGS:
|
||
return
|
||
self.flow_name_var.set(_flow_name_for_mode(self._current_mode()))
|
||
|
||
def _refresh_mode_widgets(self) -> None:
|
||
mode = self._current_mode()
|
||
is_browser = mode == target.SURFACE_BROWSER
|
||
is_desktop = mode == target.SURFACE_WINDOWS
|
||
try:
|
||
if is_browser:
|
||
self.url_row.grid()
|
||
else:
|
||
self.url_row.grid_remove()
|
||
except Exception:
|
||
pass
|
||
try:
|
||
if is_desktop:
|
||
self.desktop_options.pack(
|
||
fill=tk.X,
|
||
padx=8,
|
||
pady=(0, 4),
|
||
before=self.result_panel,
|
||
)
|
||
else:
|
||
self.desktop_options.pack_forget()
|
||
except Exception:
|
||
pass
|
||
|
||
def _synthetic_browser_info(self) -> uia.WindowInfo:
|
||
return uia.WindowInfo(
|
||
hwnd=0,
|
||
pid=0,
|
||
title="Playwright",
|
||
class_name="PlaywrightPage",
|
||
process_name="chrome.exe",
|
||
)
|
||
|
||
def _build_ui(self) -> None:
|
||
pad = {"padx": 8, "pady": (4, 2)}
|
||
|
||
top = ttk.Frame(self.root)
|
||
top.pack(fill=tk.X, padx=8, pady=(8, 2))
|
||
top.columnconfigure(1, weight=1)
|
||
ttk.Label(top, text="流程名").grid(row=0, column=0, sticky=tk.W)
|
||
ttk.Entry(top, textvariable=self.flow_name_var).grid(
|
||
row=0, column=1, sticky=tk.EW, padx=(4, 6)
|
||
)
|
||
ttk.Button(top, text="打开目录", command=self.on_open_out).grid(
|
||
row=0, column=2, padx=(0, 4)
|
||
)
|
||
self.dock_btn = ttk.Button(top, text="靠左", width=6, command=self._toggle_dock)
|
||
self.dock_btn.grid(row=0, column=3)
|
||
|
||
mode_box = ttk.LabelFrame(self.root, text="拾取模式")
|
||
mode_box.pack(fill=tk.X, **pad)
|
||
mode_box.columnconfigure(0, weight=1)
|
||
mode_row = ttk.Frame(mode_box)
|
||
mode_row.grid(row=0, column=0, sticky=tk.EW, padx=6, pady=(2, 0))
|
||
for column in range(3):
|
||
mode_row.columnconfigure(column, weight=1)
|
||
ttk.Radiobutton(
|
||
mode_row,
|
||
text="浏览器自动化",
|
||
value=target.SURFACE_BROWSER,
|
||
variable=self.mode_var,
|
||
command=self._on_mode_change,
|
||
).grid(row=0, column=0, sticky=tk.W, padx=4)
|
||
ttk.Radiobutton(
|
||
mode_row,
|
||
text="桌面自动化",
|
||
value=target.SURFACE_WINDOWS,
|
||
variable=self.mode_var,
|
||
command=self._on_mode_change,
|
||
).grid(row=0, column=1, sticky=tk.W, padx=4)
|
||
ttk.Radiobutton(
|
||
mode_row,
|
||
text="手机自动化",
|
||
value=target.SURFACE_ANDROID,
|
||
variable=self.mode_var,
|
||
command=self._on_mode_change,
|
||
).grid(row=0, column=2, sticky=tk.W, padx=4)
|
||
|
||
self.url_row = ttk.Frame(mode_box)
|
||
self.url_row.grid(row=1, column=0, sticky=tk.EW, padx=6, pady=(2, 4))
|
||
self.url_row.columnconfigure(1, weight=1)
|
||
ttk.Label(self.url_row, text="起始 URL").grid(row=0, column=0, sticky=tk.W)
|
||
ttk.Entry(self.url_row, textvariable=self.start_url_var).grid(
|
||
row=0, column=1, sticky=tk.EW, padx=(6, 0)
|
||
)
|
||
|
||
btns = ttk.LabelFrame(self.root, text="流程控制")
|
||
btns.pack(fill=tk.X, **pad)
|
||
for column in range(3):
|
||
btns.columnconfigure(column, weight=1, uniform="flow_buttons")
|
||
self.start_btn = ttk.Button(btns, text="开始", command=self.on_start_flow)
|
||
self.start_btn.grid(row=0, column=0, sticky=tk.EW, padx=(6, 2), pady=(3, 6))
|
||
self.pause_btn = ttk.Button(btns, text="暂停", command=self.on_pause_resume)
|
||
self.pause_btn.grid(row=0, column=1, sticky=tk.EW, padx=2, pady=(3, 6))
|
||
self.end_btn = ttk.Button(btns, text="结束", command=self.on_end_flow)
|
||
self.end_btn.grid(row=0, column=2, sticky=tk.EW, padx=(2, 6), pady=(3, 6))
|
||
|
||
self.desktop_options = ttk.LabelFrame(self.root, text="桌面拾取选项")
|
||
self.under_cursor_chk = ttk.Checkbutton(
|
||
self.desktop_options,
|
||
text="光标下优先（悬停、不点击）",
|
||
variable=self.under_cursor_var,
|
||
)
|
||
self.under_cursor_chk.pack(anchor=tk.W, padx=8, pady=(2, 0))
|
||
self.wecom_chk = ttk.Checkbutton(
|
||
self.desktop_options,
|
||
text="企微搜索浮层优先（SearchResultWindow2）",
|
||
variable=self.prefer_wecom_search_var,
|
||
)
|
||
self.wecom_chk.pack(anchor=tk.W, padx=8, pady=(0, 4))
|
||
|
||
self.result_panel = ttk.LabelFrame(self.root, text="运行状态与拾取结果")
|
||
self.result_panel.pack(fill=tk.BOTH, expand=True, padx=8, pady=(4, 8))
|
||
|
||
self.status_text = tk.Text(
|
||
self.result_panel,
|
||
height=3,
|
||
wrap=tk.NONE,
|
||
borderwidth=0,
|
||
padx=5,
|
||
pady=3,
|
||
)
|
||
self.status_text.pack(fill=tk.X, padx=5, pady=(3, 2))
|
||
self.status_text.configure(state=tk.DISABLED)
|
||
ttk.Separator(self.result_panel, orient=tk.HORIZONTAL).pack(
|
||
fill=tk.X, padx=5, pady=(0, 3)
|
||
)
|
||
|
||
detail_box = ttk.Frame(self.result_panel)
|
||
detail_box.pack(fill=tk.BOTH, expand=True, padx=3, pady=(0, 3))
|
||
scroll = ttk.Scrollbar(detail_box)
|
||
scroll.pack(side=tk.RIGHT, fill=tk.Y)
|
||
self.detail = tk.Text(
|
||
detail_box,
|
||
height=10,
|
||
wrap=tk.WORD,
|
||
borderwidth=0,
|
||
yscrollcommand=scroll.set,
|
||
)
|
||
self.detail.pack(fill=tk.BOTH, expand=True, padx=(3, 0), pady=3)
|
||
scroll.config(command=self.detail.yview)
|
||
self.detail.insert(
|
||
tk.END,
|
||
"1. 选择拾取模式\n"
|
||
"2. 点击「开始流程」，然后操作目标窗口\n"
|
||
"3. 完成后点击「结束流程」\n\n"
|
||
"动作、元素定位和诊断结果会显示在这里。\n",
|
||
)
|
||
self.detail.configure(state=tk.DISABLED)
|
||
self._refresh_button_states()
|
||
|
||
def _flow_is_open(self) -> bool:
|
||
return bool(self.flow_dir and self.flow and self.flow.get("status") == "open")
|
||
|
||
def _in_start_countdown(self) -> bool:
|
||
return self._countdown_after_id is not None or self._countdown_left > 0
|
||
|
||
def _on_recording_mode_change(self) -> None:
|
||
self._persist_prefs()
|
||
mode = self._current_recording_mode()
|
||
if mode == _MODE_LEGACY:
|
||
self._set_status("已选 Legacy 间隔 dump：将按约 2 秒串行写 steps（兼容旧行为）。")
|
||
elif mode == _MODE_GUIDED:
|
||
self._set_status("已选引导录制：每捕获一个动作后在本框展示定位与后置条件建议。")
|
||
elif mode == _MODE_CHECKPOINT:
|
||
self._set_status("已选手动检查点：请用「采集界面检查点」；不会自动录用户操作。")
|
||
self.auto_pick_var.set(False)
|
||
else:
|
||
self._set_status("已选自动录制：Recorder 捕获用户操作并双写 v1/v2 资产。")
|
||
|
||
def _refresh_button_states(self) -> None:
|
||
open_flow = self._flow_is_open()
|
||
counting = self._in_start_countdown()
|
||
self.end_btn.configure(state=tk.NORMAL if open_flow else tk.DISABLED)
|
||
self.start_btn.configure(
|
||
state=tk.DISABLED if (self._busy or counting or open_flow) else tk.NORMAL
|
||
)
|
||
try:
|
||
self.pause_btn.configure(
|
||
state=(
|
||
tk.NORMAL
|
||
if (open_flow and self._session and not self._busy and not counting)
|
||
else tk.DISABLED
|
||
)
|
||
)
|
||
except Exception:
|
||
pass
|
||
|
||
def _update_rec_stats(self) -> None:
|
||
if not self._session:
|
||
self.rec_stats_var.set(
|
||
"recorder: idle | actions=0 states=0 observations=0 elements=0"
|
||
)
|
||
return
|
||
st = self._session.stats()
|
||
input_health = st.get("input_backend") or {}
|
||
input_summary = ""
|
||
if input_health:
|
||
input_summary = (
|
||
f" input={input_health.get('backend', 'unknown')}"
|
||
f" raw={input_health.get('raw_input_events', 0)}"
|
||
f" mouse={input_health.get('mouse_events', 0)}"
|
||
f" keyboard={input_health.get('keyboard_events', 0)}"
|
||
f" input_dropped={input_health.get('queue_dropped', 0)}"
|
||
f" input_error={input_health.get('last_error') or '-'} |"
|
||
)
|
||
self.rec_stats_var.set(
|
||
f"recorder={st.get('recorder_status')}/{st.get('surface_recorder')} | "
|
||
f"actions={st.get('action_count')} states={st.get('state_count')} "
|
||
f"observations={st.get('observation_count')} elements={st.get('element_count')} "
|
||
f"queue={st.get('queued_events', 0)} dropped={st.get('dropped_events', 0)} | "
|
||
f"{input_summary} "
|
||
f"last={st.get('last_action_type') or '-'} "
|
||
f"target={st.get('last_target') or '-'} "
|
||
f"page={st.get('last_page_id') or '-'} "
|
||
f"obs={st.get('last_observation_id') or '-'} "
|
||
f"match={st.get('last_match_count')} unique={st.get('last_unique')}"
|
||
)
|
||
|
||
def _update_flow_label(self) -> None:
|
||
if not self.flow_dir or not self.flow:
|
||
self.flow_var.set("流程：未开始（请先点「开始流程」）")
|
||
return
|
||
if self._auto_loop_active:
|
||
auto = "LOOP" if self._is_legacy_interval() else "REC"
|
||
elif self.auto_pick_var.get() and self._flow_is_open():
|
||
auto = "armed"
|
||
else:
|
||
auto = "off"
|
||
v2 = (self.flow or {}).get("v2") or {}
|
||
live = self._session.stats() if self._session else {}
|
||
action_count = live.get("action_count", v2.get("action_count", 0))
|
||
state_count = live.get("state_count", v2.get("state_count", 0))
|
||
observation_count = live.get(
|
||
"observation_count", v2.get("observation_count", 0)
|
||
)
|
||
self.flow_var.set(
|
||
f"流程：{self.flow.get('flow_name')} | mode={self._current_mode()} | "
|
||
f"rec={self._current_recording_mode()} | "
|
||
f"status={self.flow.get('status')} | "
|
||
f"legacy_steps={self.flow.get('step_count')} "
|
||
f"actions={action_count} states={state_count} "
|
||
f"observations={observation_count} | auto={auto} | {self.flow_dir}"
|
||
)
|
||
self._update_rec_stats()
|
||
|
||
def _set_status(self, text: str) -> None:
|
||
self.status_var.set(text)
|
||
|
||
def _set_detail(self, text: str) -> None:
|
||
self.detail.configure(state=tk.NORMAL)
|
||
self.detail.delete("1.0", tk.END)
|
||
self.detail.insert(tk.END, text)
|
||
self.detail.see("1.0")
|
||
self.detail.configure(state=tk.DISABLED)
|
||
|
||
def _on_session_action_update(self, action, phase: str) -> None:
|
||
"""Snapshot an ActionRecord on a worker, then hand it safely to Tk."""
|
||
target_snapshot = dict(getattr(action, "target_snapshot", {}) or {})
|
||
evidence = dict(getattr(action, "evidence", {}) or {})
|
||
input_payload = dict(getattr(action, "input", {}) or {})
|
||
scope = dict(target_snapshot.get("window_scope") or {})
|
||
input_summary = ""
|
||
if input_payload.get("key"):
|
||
input_summary = str(input_payload.get("key"))
|
||
elif input_payload.get("text"):
|
||
value = str(input_payload.get("text"))
|
||
input_summary = value if len(value) <= 80 else value[:77] + "..."
|
||
item = {
|
||
"id": str(getattr(action, "id", "") or ""),
|
||
"sequence": int(getattr(action, "sequence", 0) or 0),
|
||
"action_type": str(getattr(action, "action_type", "") or ""),
|
||
"surface": str(getattr(action, "surface", "") or ""),
|
||
"phase": phase,
|
||
"target_name": target_snapshot.get("name")
|
||
or target_snapshot.get("automation_id")
|
||
or target_snapshot.get("semantic_name")
|
||
or target_snapshot.get("role")
|
||
or target_snapshot.get("tag")
|
||
or "-",
|
||
"target_type": target_snapshot.get("control_type")
|
||
or target_snapshot.get("role")
|
||
or target_snapshot.get("tag")
|
||
or "-",
|
||
"window_title": scope.get("title") or "",
|
||
"window_class": scope.get("class_name") or "",
|
||
"input_summary": input_summary,
|
||
"locator": evidence.get("primary_locator") or "",
|
||
"match_count": evidence.get("locator_match_count"),
|
||
"unique": evidence.get("locator_unique"),
|
||
"error": evidence.get("eventual_evidence_error")
|
||
or evidence.get("error")
|
||
or "",
|
||
}
|
||
self._post_ui(self._apply_live_action_update, item)
|
||
|
||
def _apply_live_action_update(self, item: dict) -> None:
|
||
updated = _merge_current_live_action(self._current_live_action, item)
|
||
if updated is self._current_live_action:
|
||
return
|
||
self._current_live_action = updated
|
||
assert updated is not None
|
||
self.target_var.set(
|
||
f"当前动作：#{updated.get('sequence')} {updated.get('action_type')} · "
|
||
f"{updated.get('phase')}"
|
||
)
|
||
self._set_detail("当前步骤\n\n" + _format_live_action(updated))
|
||
|
||
def _show_picking_placeholder(self, *, auto: bool) -> None:
|
||
n = int((self.flow or {}).get("step_count") or 0) + 1
|
||
mode = "自动" if auto else "检查点"
|
||
self.target_var.set(f"{mode}采集中…（即将写入第 {n} 步）")
|
||
self._set_status(f"正在{mode}采集：已清空本框上次结果，请稍候…")
|
||
self._set_detail(
|
||
"（已清空上次本步结果）\n\n"
|
||
f"正在读取控件树，完成后写入当前流程第 {n} 步…\n"
|
||
)
|
||
|
||
def _exclude_hwnds(self) -> set[int]:
|
||
try:
|
||
return {int(self.root.winfo_id())}
|
||
except Exception:
|
||
return set()
|
||
|
||
def _cancel_auto_timer(self) -> None:
|
||
if self._auto_after_id is not None:
|
||
try:
|
||
self.root.after_cancel(self._auto_after_id)
|
||
except Exception:
|
||
pass
|
||
self._auto_after_id = None
|
||
|
||
def _cancel_countdown(self) -> None:
|
||
if self._countdown_after_id is not None:
|
||
try:
|
||
self.root.after_cancel(self._countdown_after_id)
|
||
except Exception:
|
||
pass
|
||
self._countdown_after_id = None
|
||
self._countdown_left = 0
|
||
|
||
def _cancel_heartbeat(self) -> None:
|
||
if self._heartbeat_after_id is not None:
|
||
try:
|
||
self.root.after_cancel(self._heartbeat_after_id)
|
||
except Exception:
|
||
pass
|
||
self._heartbeat_after_id = None
|
||
|
||
def _stop_auto_loop(self) -> None:
|
||
self._auto_loop_active = False
|
||
self._cancel_auto_timer()
|
||
self._cancel_countdown()
|
||
self._cancel_heartbeat()
|
||
|
||
def _stop_session(self) -> None:
|
||
if self._session is not None:
|
||
try:
|
||
self._session.stop()
|
||
except Exception:
|
||
pass
|
||
self._session = None
|
||
self._rec_paused = False
|
||
try:
|
||
self.pause_btn.configure(text="暂停")
|
||
except Exception:
|
||
pass
|
||
|
||
def _schedule_heartbeat(self) -> None:
|
||
self._cancel_heartbeat()
|
||
if not self._auto_loop_active or not self._flow_is_open():
|
||
return
|
||
if self._is_legacy_interval():
|
||
return
|
||
self._heartbeat_after_id = self.root.after(_HEARTBEAT_MS, self._heartbeat_tick)
|
||
|
||
def _heartbeat_tick(self) -> None:
|
||
self._heartbeat_after_id = None
|
||
if not self._auto_loop_active or not self._flow_is_open():
|
||
return
|
||
if self._session and not self._rec_paused:
|
||
try:
|
||
self._session.heartbeat()
|
||
except Exception:
|
||
pass
|
||
self._update_flow_label()
|
||
# Guided mode: refresh last action detail
|
||
if self._current_recording_mode() == _MODE_GUIDED:
|
||
self._show_guided_last_action()
|
||
self._schedule_heartbeat()
|
||
|
||
def _show_guided_last_action(self) -> None:
|
||
if not self._session:
|
||
return
|
||
act = self._session.manager.last_action
|
||
if not act:
|
||
return
|
||
lines = [
|
||
f"最近动作 #{act.sequence} {act.action_type} status={act.status}",
|
||
f"target: {(act.target_snapshot or {}).get('name') or (act.target_snapshot or {}).get('automation_id')}",
|
||
f"scope: {(act.target_snapshot or {}).get('window_scope')}",
|
||
f"primary locator: {(act.evidence or {}).get('primary_locator')}",
|
||
f"match count: {(act.evidence or {}).get('locator_match_count')} unique={(act.evidence or {}).get('locator_unique')}",
|
||
"proposed postconditions:",
|
||
]
|
||
for p in (act.proposed_postconditions or [])[:8]:
|
||
lines.append(f" - [{p.get('status')}] {p.get('description')}")
|
||
if self._session.manager.last_warning:
|
||
lines.append(f"warning: {self._session.manager.last_warning}")
|
||
self._set_detail("\n".join(lines))
|
||
|
||
def _make_dump_fn(self):
|
||
"""Build a dump callable for SnapshotService (runs on worker threads)."""
|
||
|
||
# Snapshot every Tk-owned value on the main thread. The returned
|
||
# callable is deliberately free of Tk calls so it cannot participate
|
||
# in a Tk <-> recorder lock inversion.
|
||
mode = self._current_mode()
|
||
under_cursor = bool(self.under_cursor_var.get())
|
||
prefer_search = bool(self.prefer_wecom_search_var.get())
|
||
sticky_pid = self.sticky_pid
|
||
exclude_hwnds = self._exclude_hwnds()
|
||
max_depth = self.max_depth
|
||
max_nodes = self.max_nodes
|
||
warmed_pids = self._warmed_pids
|
||
|
||
def dump_fn(
|
||
*, page_id: str = "", document_id: str = "", frame_id: str = "",
|
||
target_hwnd: int = 0, target_pid: int = 0,
|
||
target_class_name: str = "", target_title: str = "",
|
||
):
|
||
uia.ensure_com_sta()
|
||
surface = mode
|
||
sel_meta: dict = {"pick_mode": mode, "sticky_pid": sticky_pid}
|
||
if surface == target.SURFACE_BROWSER:
|
||
info = self._synthetic_browser_info()
|
||
sel_meta["strategy"] = "playwright_session"
|
||
elif target_hwnd:
|
||
try:
|
||
info = uia.window_info_from_hwnd(int(target_hwnd))
|
||
except Exception:
|
||
info = None
|
||
if info is None and target_class_name == "SearchResultWindow2":
|
||
info = uia.find_wecom_search_result_window(
|
||
pid=int(target_pid or 0) or None,
|
||
exclude_hwnds=exclude_hwnds,
|
||
)
|
||
if info is None:
|
||
raise RuntimeError(
|
||
f"event window disappeared: hwnd={target_hwnd} class={target_class_name}"
|
||
)
|
||
cands = [info]
|
||
sel_meta.update(
|
||
{
|
||
"strategy": "event_window",
|
||
"chosen_hwnd": int(info.hwnd),
|
||
"chosen_class": info.class_name,
|
||
"event_title": target_title,
|
||
}
|
||
)
|
||
sel_meta["candidate_count"] = 1
|
||
else:
|
||
info, cands, win_meta = uia.resolve_capture_target(
|
||
exclude_hwnds=exclude_hwnds,
|
||
prefer_pid=sticky_pid,
|
||
under_cursor=under_cursor,
|
||
prefer_wecom_search_popup=prefer_search,
|
||
)
|
||
sel_meta.update(win_meta)
|
||
sel_meta["candidate_count"] = len(cands)
|
||
driver = get_driver(surface)
|
||
warm_key = (
|
||
(int(info.pid), str(info.class_name or ""))
|
||
if surface == target.SURFACE_WINDOWS
|
||
else int(info.pid or 0)
|
||
)
|
||
warmup_mode = "light" if warm_key in warmed_pids else "full"
|
||
if surface == target.SURFACE_WINDOWS:
|
||
# Recording already has a persisted event-time element atlas.
|
||
# Evidence may probe the full tree, but must never run a
|
||
# blocking product wake in the middle of user input.
|
||
warmup_mode = "probe"
|
||
if target_class_name == "SearchResultWindow2":
|
||
# A search popup may live for only a few seconds. Persist its
|
||
# bounded tree immediately; heavyweight kit wake/FindAll remains
|
||
# supplementary and must not make the event-time snapshot miss it.
|
||
warmup_mode = "off"
|
||
# Recorder path: after-action dumps may warm; click capture itself never warms.
|
||
dump_kwargs = dict(
|
||
info=info,
|
||
selection_meta=sel_meta,
|
||
max_depth=(12 if target_class_name == "SearchResultWindow2" else max_depth),
|
||
max_nodes=(1500 if target_class_name == "SearchResultWindow2" else max_nodes),
|
||
warmup_mode=warmup_mode,
|
||
warmed_pids=warmed_pids,
|
||
)
|
||
if surface == target.SURFACE_BROWSER:
|
||
dump_kwargs.update(
|
||
page_id=page_id,
|
||
document_id=document_id,
|
||
frame_id=frame_id,
|
||
)
|
||
result = driver.dump(**dump_kwargs)
|
||
if info.pid:
|
||
warmed_pids.add(warm_key)
|
||
return result
|
||
|
||
return dump_fn
|
||
|
||
def _start_recording_session(
|
||
self, *, entry_dump=None, checkpoint_only: bool = False
|
||
) -> None:
|
||
if not self.flow_dir or not self.flow:
|
||
return
|
||
rec_mode = _MODE_CHECKPOINT if checkpoint_only else self._current_recording_mode()
|
||
session = FlowSession(
|
||
self.flow_dir,
|
||
flow_name=str(self.flow.get("flow_name") or "flow"),
|
||
surface=self._current_mode(),
|
||
recording_mode=rec_mode,
|
||
dump_fn=self._make_dump_fn(),
|
||
exclude_hwnds=self._exclude_hwnds,
|
||
on_action_update=self._on_session_action_update,
|
||
stability_timeout_sec=_STABILITY_TIMEOUT_SEC,
|
||
)
|
||
self._session = session
|
||
if rec_mode == _MODE_LEGACY:
|
||
self._auto_loop_active = True
|
||
self._schedule_auto_next(_AUTO_START_DELAY_MS)
|
||
return
|
||
|
||
self._busy = True
|
||
self._refresh_button_states()
|
||
self._set_status("正在后台采集入口状态，界面仍可响应……")
|
||
|
||
def work() -> None:
|
||
try:
|
||
if rec_mode == _MODE_CHECKPOINT:
|
||
session.manager.action_flow.recording_mode = _MODE_CHECKPOINT
|
||
session.manager.start(entry_dump=entry_dump)
|
||
session._refresh_stats()
|
||
else:
|
||
session.start(entry_dump=entry_dump)
|
||
stats = session.stats()
|
||
if int(stats.get("state_count") or 0) == 0:
|
||
warning = str(stats.get("warning") or "入口状态采集没有产物")
|
||
raise RuntimeError(f"ERROR:ENTRY_CAPTURE_FAILED {warning}")
|
||
except Exception as exc:
|
||
self._post_ui(self._on_recording_session_started, session, rec_mode, exc)
|
||
try:
|
||
session.stop()
|
||
except Exception:
|
||
pass
|
||
return
|
||
self._post_ui(self._on_recording_session_started, session, rec_mode, None)
|
||
|
||
threading.Thread(
|
||
target=work, name="ui-inspector-session-start", daemon=True
|
||
).start()
|
||
|
||
def _on_recording_session_started(
|
||
self, session: FlowSession, rec_mode: str, error: BaseException | None
|
||
) -> None:
|
||
if self._session is not session:
|
||
return
|
||
self._busy = False
|
||
if error is not None:
|
||
self._session = None
|
||
self._set_status(f"录制会话启动失败: {error}")
|
||
self._refresh_button_states()
|
||
return
|
||
self._auto_loop_active = True
|
||
self._rec_paused = False
|
||
self._update_flow_label()
|
||
self._refresh_button_states()
|
||
if rec_mode == _MODE_CHECKPOINT:
|
||
self._auto_loop_active = False
|
||
self._set_status("入口状态已采集；可手动添加界面检查点。")
|
||
else:
|
||
self.target_var.set("动作录制中：等待用户操作")
|
||
self._set_status("入口状态已采集，动作录制已就绪。")
|
||
self._schedule_heartbeat()
|
||
|
||
def _start_auto_loop(self, *, first_delay_ms: int = _AUTO_START_DELAY_MS) -> None:
|
||
"""Legacy interval loop OR action-driven recording start."""
|
||
self._pending_end = False
|
||
if self._is_legacy_interval():
|
||
self._auto_loop_active = True
|
||
self._update_flow_label()
|
||
self._refresh_button_states()
|
||
self._schedule_auto_next(first_delay_ms)
|
||
return
|
||
# Action-driven: the desktop baseline is intentionally lightweight;
|
||
# full provider evidence is collected asynchronously after actions.
|
||
entry_dump = (
|
||
self._desktop_entry_dump
|
||
if self._current_mode() == target.SURFACE_WINDOWS
|
||
else None
|
||
)
|
||
self._start_recording_session(entry_dump=entry_dump)
|
||
|
||
def _set_inspector_topmost(self, enabled: bool) -> None:
|
||
try:
|
||
self.root.attributes("-topmost", bool(enabled))
|
||
except Exception:
|
||
pass
|
||
|
||
def _ensure_inspector_visible(self) -> None:
|
||
"""Keep panel on-screen for progress; do not force-focus (avoids stealing FG)."""
|
||
try:
|
||
if str(self.root.state()) == "iconic":
|
||
self.root.deiconify()
|
||
except Exception:
|
||
pass
|
||
|
||
def _arm_auto_with_countdown(self, *, seconds: int = _START_COUNTDOWN_SEC) -> None:
|
||
"""Visible countdown, then enter mode-specific auto loop."""
|
||
if not self._flow_is_open():
|
||
return
|
||
# Stop prior loop/timers without clearing the countdown we are about to arm.
|
||
self._auto_loop_active = False
|
||
self._cancel_auto_timer()
|
||
self._cancel_countdown()
|
||
self._ensure_inspector_visible()
|
||
# Stay topmost for progress; capture still uses focus + exclude self.
|
||
self._set_inspector_topmost(True)
|
||
self._countdown_left = max(1, int(seconds))
|
||
self._update_flow_label()
|
||
mode = self._current_mode()
|
||
if mode == target.SURFACE_BROWSER:
|
||
url = (self.start_url_var.get() or "").strip()
|
||
behavior = (
|
||
"Legacy：打开浏览器后每 2 秒采集一次"
|
||
if self._is_legacy_interval()
|
||
else "打开浏览器后监听用户动作，并采集动作后的稳定状态"
|
||
)
|
||
self._set_status(
|
||
f"倒计时 {self._countdown_left} 秒：将打开浏览器并进入 {url or '(起始 URL)'}"
|
||
)
|
||
self._set_detail(
|
||
f"流程：{self.flow_dir}\n\n"
|
||
f"- 模式：浏览器自动化\n"
|
||
f"- 起始 URL：{url or '(空，自行在浏览器中打开目标站)'}\n"
|
||
f"- 倒计时结束后：{behavior}\n"
|
||
f"- 可点「结束流程」或取消「自动拾取」中止\n"
|
||
)
|
||
else:
|
||
behavior = (
|
||
"Legacy：识别窗口后每 2 秒采集一次"
|
||
if self._is_legacy_interval()
|
||
else "识别窗口后监听用户动作，并采集动作后的稳定状态"
|
||
)
|
||
self._set_status(
|
||
f"倒计时 {self._countdown_left} 秒：请把要采集的桌面窗口点到最前端"
|
||
"（本面板置顶看进度；不强制放大目标窗）。"
|
||
)
|
||
self._set_detail(
|
||
f"流程：{self.flow_dir}\n\n"
|
||
f"- 模式：桌面自动化\n"
|
||
f"- {self._countdown_left} 秒内请点击目标窗口（给它焦点）\n"
|
||
f"- 倒计时结束后：{behavior}\n"
|
||
f"- 可点「结束流程」或取消「自动拾取」中止\n"
|
||
)
|
||
self._refresh_button_states()
|
||
self._countdown_after_id = self.root.after(1000, self._countdown_tick)
|
||
|
||
def _countdown_tick(self) -> None:
|
||
self._countdown_after_id = None
|
||
if not self._flow_is_open():
|
||
self._countdown_left = 0
|
||
self._refresh_button_states()
|
||
return
|
||
if not self.auto_pick_var.get():
|
||
self._countdown_left = 0
|
||
self._refresh_button_states()
|
||
self._set_status("已取消自动拾取倒计时；可手动点「拾取当前页」。")
|
||
return
|
||
self._countdown_left -= 1
|
||
if self._countdown_left > 0:
|
||
mode = self._current_mode()
|
||
if mode == target.SURFACE_BROWSER:
|
||
self._set_status(
|
||
f"倒计时 {self._countdown_left} 秒：准备打开浏览器…"
|
||
)
|
||
else:
|
||
self._set_status(
|
||
f"倒计时 {self._countdown_left} 秒：请把要采集的窗口点到最前端…"
|
||
)
|
||
self._refresh_button_states()
|
||
self._countdown_after_id = self.root.after(1000, self._countdown_tick)
|
||
return
|
||
self._countdown_left = 0
|
||
self._refresh_button_states()
|
||
if self._current_mode() == target.SURFACE_BROWSER:
|
||
self._set_status("倒计时结束，正在打开浏览器…")
|
||
self._begin_auto_browser()
|
||
else:
|
||
self._set_status("倒计时结束，正在识别最前端窗口…")
|
||
self._begin_auto_with_probe()
|
||
|
||
def _begin_auto_browser(self) -> None:
|
||
"""Launch Playwright with start URL, then enter auto loop (no Win32 probe)."""
|
||
if self._busy:
|
||
return
|
||
self._busy = True
|
||
self._auto_loop_active = False
|
||
self._refresh_button_states()
|
||
self._set_inspector_topmost(True)
|
||
self._ensure_inspector_visible()
|
||
url = (self.start_url_var.get() or "").strip()
|
||
status_url = url or "(未填起始 URL，打开空白页后可自行导航)"
|
||
self._set_status(f"浏览器模式：正在打开 {status_url} …")
|
||
self._set_detail(
|
||
f"启动 Playwright…\n\n"
|
||
f"- 起始 URL：{url or '(空)'}\n"
|
||
f"- profile：{kit_browser.profile_dir()}\n"
|
||
f"- 打开完成后进入{'Legacy 间隔采集' if self._is_legacy_interval() else '动作录制'}\n"
|
||
)
|
||
|
||
def work() -> None:
|
||
try:
|
||
# 有 URL 才 force_goto；空则只启浏览器
|
||
sess = kit_browser.ensure_session(
|
||
start_url=url or "about:blank",
|
||
force_goto=bool(url),
|
||
)
|
||
self._post_ui(self._on_browser_ready, sess, url)
|
||
except Exception as exc:
|
||
self._post_ui(self._on_frontmost_fail, exc)
|
||
|
||
threading.Thread(target=work, daemon=True).start()
|
||
|
||
def _on_browser_ready(self, sess: dict, url: str) -> None:
|
||
self._busy = False
|
||
self._refresh_button_states()
|
||
live = str(sess.get("url") or "about:blank")
|
||
if url and (not live or live == "about:blank" or live.startswith("about:")):
|
||
self._on_frontmost_fail(
|
||
RuntimeError(
|
||
f"ERROR:NAVIGATION_FAILED requested={url!r} actual={live!r}"
|
||
)
|
||
)
|
||
return
|
||
navigation = dict(sess.get("navigation") or {})
|
||
nav_warning = str(navigation.get("warning") or "")
|
||
self._set_status(
|
||
f"浏览器已就绪：{live} — "
|
||
+ ("开始 Legacy 间隔采集。" if self._is_legacy_interval() else "正在采集入口状态。")
|
||
)
|
||
self._set_detail(
|
||
f"采集目标（Playwright）:\n"
|
||
f" mode=browser\n"
|
||
f" start_url={url or '(空)'}\n"
|
||
f" live_url={live}\n"
|
||
f" profile={sess.get('profile_dir')}\n"
|
||
f" reused={sess.get('reused')}\n\n"
|
||
+ (f"导航警告：{nav_warning}\n\n" if nav_warning else "")
|
||
+ (
|
||
"即将进入 Legacy 自动循环（步间 2 秒）…\n"
|
||
if self._is_legacy_interval()
|
||
else "即将启动动作录制器…\n"
|
||
)
|
||
+ (
|
||
""
|
||
if url
|
||
else "未填起始 URL：请在弹出的浏览器中打开目标站（空页会自动跳过写入）。\n"
|
||
)
|
||
)
|
||
self._set_inspector_topmost(True)
|
||
self._ensure_inspector_visible()
|
||
self._start_auto_loop(first_delay_ms=_AUTO_START_DELAY_MS)
|
||
|
||
def _begin_auto_with_probe(self) -> None:
|
||
"""Desktop mode: resolve frontmost target, warmup best-effort, then loop."""
|
||
if self._busy:
|
||
return
|
||
self._busy = True
|
||
self._auto_loop_active = False
|
||
self._refresh_button_states()
|
||
# Keep topmost for progress; target = focused window (Inspector hwnd excluded).
|
||
self._set_inspector_topmost(True)
|
||
self._ensure_inspector_visible()
|
||
self._set_status(
|
||
"桌面模式：正在识别最前端窗口（焦点窗=采集目标，已排除本面板）…"
|
||
)
|
||
self._set_detail(
|
||
"识别最前端窗口中…\n\n"
|
||
"- 模式：桌面自动化（强制 UIA）\n"
|
||
"- 规则：谁有焦点就采谁（与窗口大小无关）\n"
|
||
"- 若是企微：自动调用 kit ensure_ui_ready\n"
|
||
)
|
||
exclude_hwnds = self._exclude_hwnds()
|
||
|
||
def work() -> None:
|
||
try:
|
||
uia.ensure_com_sta()
|
||
report = uia.prepare_frontmost_for_capture(
|
||
exclude_hwnds=exclude_hwnds,
|
||
timeout_sec=_FRONTMOST_WAIT_SEC,
|
||
)
|
||
self._post_ui(self._on_frontmost_ready, report)
|
||
except Exception as exc:
|
||
self._post_ui(self._on_frontmost_fail, exc)
|
||
|
||
threading.Thread(target=work, daemon=True).start()
|
||
|
||
def _on_frontmost_ready(self, report: dict) -> None:
|
||
self._busy = False
|
||
self._refresh_button_states()
|
||
win = report.get("window") or {}
|
||
if win.get("pid"):
|
||
self.sticky_pid = int(win["pid"])
|
||
bounds = dict(win.get("rect") or win.get("bounds") or {})
|
||
root_row = {
|
||
"name": str(win.get("title") or ""),
|
||
"value": "",
|
||
"automation_id": "",
|
||
"control_type": "WindowControl",
|
||
"class_name": str(win.get("class_name") or ""),
|
||
"framework_id": "Win32",
|
||
"is_enabled": True,
|
||
"is_offscreen": False,
|
||
"bounds": bounds,
|
||
"window_scope": dict(win),
|
||
}
|
||
self._desktop_entry_dump = DumpResult(
|
||
surface=target.SURFACE_WINDOWS,
|
||
driver="uiautomation-light",
|
||
window=dict(win),
|
||
tree={**root_row, "children": []},
|
||
index=[dict(root_row)],
|
||
locatable=[dict(root_row)],
|
||
stats={
|
||
"capture_kind": "entry_light",
|
||
"nodes_kept": 1,
|
||
"locatable_count": 1,
|
||
"provider_enrichment": "deferred",
|
||
},
|
||
selection={
|
||
"strategy": str(report.get("strategy") or "foreground_only"),
|
||
"entry_light": True,
|
||
},
|
||
use_hwnd_screenshot=False,
|
||
)
|
||
import kit_wake
|
||
|
||
kit = report.get("kit_wake") or {}
|
||
kit_line = kit_wake.format_kit_wake_line(kit)
|
||
sparse_note = ""
|
||
if report.get("provider_cold") or report.get("uia_sparse"):
|
||
if kit.get("reason") == "kit_not_installed":
|
||
sparse_note = (
|
||
f"（仍偏稀 FindAll={report.get('findall')}：未安装 platform-kit，"
|
||
"企微无法走 n2-wake）"
|
||
)
|
||
elif kit.get("applicable") and not kit.get("ok"):
|
||
sparse_note = (
|
||
f"（kit 暖树未成功 FindAll={report.get('findall')} "
|
||
f"code={kit.get('error_code')}；仍按此窗采集）"
|
||
)
|
||
else:
|
||
sparse_note = (
|
||
f"（UIA 仍偏稀 FindAll={report.get('findall')}；仍按此前端窗采集）"
|
||
)
|
||
self._set_status(
|
||
f"已锁定桌面窗：{win.get('title')!r} "
|
||
f"findall={report.get('findall')} — "
|
||
+ ("开始 Legacy 间隔采集。" if self._is_legacy_interval() else "正在采集入口状态。")
|
||
+ sparse_note
|
||
)
|
||
self._set_detail(
|
||
f"采集目标（桌面 UIA）:\n"
|
||
f" mode=windows\n"
|
||
f" title={win.get('title')!r}\n"
|
||
f" class={win.get('class_name')!r}\n"
|
||
f" hwnd={win.get('hwnd')} pid={win.get('pid')}\n"
|
||
f" findall={report.get('findall')} sparse={report.get('uia_sparse')} "
|
||
f"cold={report.get('provider_cold')}\n"
|
||
f" {kit_line}\n\n"
|
||
+ (
|
||
"即将进入 Legacy 自动循环（步间 2 秒）…\n"
|
||
if self._is_legacy_interval()
|
||
else "即将启动动作录制器…\n"
|
||
)
|
||
)
|
||
self._set_inspector_topmost(True)
|
||
self._ensure_inspector_visible()
|
||
self._start_auto_loop(first_delay_ms=_AUTO_START_DELAY_MS)
|
||
|
||
def _on_frontmost_fail(self, exc: BaseException) -> None:
|
||
self._busy = False
|
||
self._auto_loop_active = False
|
||
# Keep auto checked (default on); user can retry or end flow.
|
||
self._set_inspector_topmost(True)
|
||
self._ensure_inspector_visible()
|
||
self._refresh_button_states()
|
||
self._set_status(f"未能锁定最前端目标: {exc}")
|
||
messagebox.showerror("UI Inspector", f"未能锁定最前端目标:\n{exc}")
|
||
|
||
def _schedule_auto_next(self, delay_ms: int) -> None:
|
||
self._cancel_auto_timer()
|
||
if not self._auto_loop_active or not self._flow_is_open():
|
||
return
|
||
self._auto_after_id = self.root.after(delay_ms, self._auto_loop_tick)
|
||
|
||
def _auto_loop_tick(self) -> None:
|
||
self._auto_after_id = None
|
||
if not self._auto_loop_active or not self._flow_is_open():
|
||
return
|
||
if self._busy:
|
||
# Should not happen in serial design; re-check soon.
|
||
self._schedule_auto_next(200)
|
||
return
|
||
self.on_pick_page(auto=True)
|
||
|
||
def _continue_auto_loop_after_pick(self) -> None:
|
||
if self._pending_end:
|
||
self._pending_end = False
|
||
self._stop_auto_loop()
|
||
self._finish_end_flow()
|
||
return
|
||
if self._auto_loop_active and self._flow_is_open():
|
||
self._set_status(
|
||
f"自动循环：第 {self.flow.get('step_count') if self.flow else '?'} 步已处理；"
|
||
f"{_AUTO_GAP_MS // 1000} 秒后开始下一拍。点「结束流程」退出循环。"
|
||
)
|
||
self._schedule_auto_next(_AUTO_GAP_MS)
|
||
|
||
def on_auto_toggle(self) -> None:
|
||
if self.auto_pick_var.get():
|
||
if (
|
||
self._flow_is_open()
|
||
and not self._auto_loop_active
|
||
and not self._busy
|
||
and not self._in_start_countdown()
|
||
):
|
||
self._arm_auto_with_countdown()
|
||
else:
|
||
self._update_flow_label()
|
||
else:
|
||
was = self._auto_loop_active or self._in_start_countdown()
|
||
self._stop_auto_loop()
|
||
self._update_flow_label()
|
||
self._refresh_button_states()
|
||
if was and self._flow_is_open():
|
||
self._set_status("已停止自动录制/倒计时；仍可手动点「采集界面检查点」。")
|
||
elif self._flow_is_open():
|
||
self._set_status("自动录制已关闭；可手动点「采集界面检查点」。")
|
||
|
||
def on_open_out(self) -> None:
|
||
path = self.flow_dir if self.flow_dir else self.captures_root
|
||
path.mkdir(parents=True, exist_ok=True)
|
||
try:
|
||
os.startfile(str(path)) # type: ignore[attr-defined]
|
||
except Exception:
|
||
subprocess.Popen(["explorer", str(path)])
|
||
|
||
def on_start_flow(self) -> None:
|
||
if self._busy:
|
||
return
|
||
mode = self._current_mode()
|
||
if mode == target.SURFACE_ANDROID:
|
||
messagebox.showwarning(
|
||
"UI Inspector",
|
||
"手机自动化尚未实现。\n请先选择「浏览器自动化」或「桌面自动化」。",
|
||
)
|
||
return
|
||
self._persist_prefs()
|
||
|
||
if self.flow and self.flow.get("status") == "open":
|
||
if not messagebox.askyesno(
|
||
"UI Inspector",
|
||
"当前已有进行中的流程。\n是否结束旧流程并开始新流程？",
|
||
):
|
||
return
|
||
self._stop_auto_loop()
|
||
self._pending_end = False
|
||
try:
|
||
art.end_flow(self.flow_dir, self.flow) # type: ignore[arg-type]
|
||
except Exception:
|
||
pass
|
||
self._stop_session()
|
||
|
||
name = (self.flow_name_var.get() or "").strip()
|
||
if not name:
|
||
default_name = _flow_name_for_mode(mode)
|
||
name = simpledialog.askstring(
|
||
"流程名", "请输入流程名称：", initialvalue=default_name
|
||
)
|
||
if not name:
|
||
return
|
||
self.flow_name_var.set(name)
|
||
|
||
try:
|
||
self.flow_dir = art.create_flow(
|
||
self.captures_root,
|
||
flow_name=name,
|
||
recording_mode=self._current_recording_mode(),
|
||
)
|
||
self.flow = art.load_flow(self.flow_dir)
|
||
self._pending_end = False
|
||
self._warmed_pids.clear()
|
||
self._current_live_action = None
|
||
self._desktop_entry_dump = None
|
||
self._stop_session()
|
||
self._update_flow_label()
|
||
self._refresh_button_states()
|
||
if self._current_recording_mode() == _MODE_CHECKPOINT:
|
||
self.auto_pick_var.set(False)
|
||
if self.auto_pick_var.get() and self._current_recording_mode() != _MODE_CHECKPOINT:
|
||
# No modal / no minimize: keep panel visible for countdown + progress.
|
||
self._arm_auto_with_countdown(seconds=_START_COUNTDOWN_SEC)
|
||
else:
|
||
self._stop_auto_loop()
|
||
self._ensure_inspector_visible()
|
||
if mode == target.SURFACE_BROWSER:
|
||
url = (self.start_url_var.get() or "").strip()
|
||
self._set_status(
|
||
f"已开始流程：{self.flow_dir.name}。"
|
||
+ (
|
||
f"将打开 {url}；请点「采集界面检查点」。"
|
||
if url
|
||
else "已启浏览器（无起始 URL）；请自行打开目标站后点「采集界面检查点」。"
|
||
)
|
||
)
|
||
# Eager open browser so manual pick is ready
|
||
def _open() -> None:
|
||
try:
|
||
kit_browser.ensure_session(
|
||
start_url=url or "about:blank",
|
||
force_goto=bool(url),
|
||
)
|
||
except Exception:
|
||
pass
|
||
|
||
threading.Thread(target=_open, daemon=True).start()
|
||
else:
|
||
self._set_status(
|
||
f"已开始流程：{self.flow_dir.name}。请翻到目标页后点「采集界面检查点」。"
|
||
)
|
||
# Manual mode owns no surface recorder; it only keeps the v2
|
||
# manager available for explicit checkpoints.
|
||
self._start_recording_session(checkpoint_only=True)
|
||
self._set_detail(
|
||
f"流程已创建：{self.flow_dir}\n\n"
|
||
f"- 模式：{mode}\n"
|
||
f"- 录制：{self._current_recording_mode()}\n"
|
||
f"- flow.json / action_flow.json / element_catalog.json / steps/\n"
|
||
f"- 手动点「采集界面检查点」；或勾选自动录制用户操作。\n"
|
||
)
|
||
except Exception as exc:
|
||
messagebox.showerror("UI Inspector", str(exc))
|
||
|
||
def _finish_end_flow(self) -> None:
|
||
if not self.flow_dir or not self.flow:
|
||
return
|
||
if self.flow.get("status") != "open":
|
||
self._refresh_button_states()
|
||
return
|
||
session = self._session
|
||
flow_dir = self.flow_dir
|
||
flow = self.flow
|
||
self._session = None
|
||
self._busy = True
|
||
self._refresh_button_states()
|
||
self._set_status("正在后台刷新动作队列并结束流程……")
|
||
|
||
def work() -> None:
|
||
try:
|
||
if session is not None:
|
||
session.stop()
|
||
latest_flow = art.load_flow(flow_dir)
|
||
ended = art.end_flow(flow_dir, latest_flow)
|
||
self._post_ui(self._on_end_flow_finished, ended, None)
|
||
except Exception as exc:
|
||
self._post_ui(self._on_end_flow_finished, None, exc)
|
||
|
||
threading.Thread(
|
||
target=work, name="ui-inspector-session-stop", daemon=True
|
||
).start()
|
||
|
||
def _on_end_flow_finished(
|
||
self, ended_flow: dict | None, error: BaseException | None
|
||
) -> None:
|
||
self._busy = False
|
||
if error is not None or ended_flow is None:
|
||
self._refresh_button_states()
|
||
messagebox.showerror("UI Inspector", str(error or "结束流程失败"))
|
||
return
|
||
try:
|
||
self.flow = ended_flow
|
||
self._update_flow_label()
|
||
self._refresh_button_states()
|
||
n = self.flow.get("step_count") or 0
|
||
v2 = self.flow.get("v2") or {}
|
||
self._set_status(
|
||
f"流程已结束：actions={v2.get('action_count', '?')}，"
|
||
f"states={v2.get('state_count', '?')}，"
|
||
f"observations={v2.get('observation_count', '?')}，"
|
||
f"legacy_steps={n}。"
|
||
"可再点「开始流程」开新文档。"
|
||
)
|
||
self._set_detail(
|
||
f"流程已结束：{self.flow_dir}\n\n"
|
||
f"- v1 steps: {n}\n"
|
||
f"- v2 actions: {v2.get('action_count')}\n"
|
||
f"- 请打开 SUMMARY.md / action_flow.json / element_catalog.json\n"
|
||
)
|
||
messagebox.showinfo(
|
||
"UI Inspector",
|
||
f"流程已结束\nactions={v2.get('action_count')} "
|
||
f"states={v2.get('state_count')} "
|
||
f"observations={v2.get('observation_count')} "
|
||
f"legacy_steps={n}\n{self.flow_dir}",
|
||
)
|
||
except Exception as exc:
|
||
messagebox.showerror("UI Inspector", str(exc))
|
||
|
||
def on_pause_resume(self) -> None:
|
||
if not self._session or not self._flow_is_open():
|
||
return
|
||
if self._rec_paused:
|
||
self._session.resume()
|
||
self._rec_paused = False
|
||
self.pause_btn.configure(text="暂停")
|
||
self._set_status("已继续录制。")
|
||
else:
|
||
self._session.pause()
|
||
self._rec_paused = True
|
||
self.pause_btn.configure(text="继续")
|
||
self._set_status("已暂停录制（Inspector 自身操作不会写入）。")
|
||
self._update_rec_stats()
|
||
|
||
def on_end_flow(self) -> None:
|
||
if not self.flow_dir or not self.flow:
|
||
messagebox.showwarning("UI Inspector", "没有进行中的流程。")
|
||
return
|
||
if self.flow.get("status") != "open":
|
||
messagebox.showinfo("UI Inspector", "流程已经结束。")
|
||
return
|
||
|
||
self._stop_auto_loop()
|
||
# Keep「自动录制」勾选状态（默认开），方便下一轮直接开始。
|
||
|
||
if self._busy:
|
||
self._pending_end = True
|
||
self._set_status("已请求结束：等待当前这一拍 dump/保存完成…")
|
||
self._refresh_button_states()
|
||
return
|
||
|
||
self._pending_end = False
|
||
self._finish_end_flow()
|
||
|
||
def on_pick_page(self, *, auto: bool = False) -> None:
|
||
"""Manual checkpoint or legacy auto dump. Action-driven mode uses Recorder."""
|
||
if self._busy:
|
||
return
|
||
if not self._flow_is_open():
|
||
if not auto:
|
||
messagebox.showwarning("UI Inspector", "请先点「开始流程」。")
|
||
return
|
||
if auto and not self._auto_loop_active:
|
||
return
|
||
# In action-driven mode, auto ticks should not dump steps.
|
||
if auto and not self._is_legacy_interval():
|
||
return
|
||
|
||
self._busy = True
|
||
self._refresh_button_states()
|
||
self._show_picking_placeholder(auto=auto)
|
||
self.root.update_idletasks()
|
||
|
||
# Capture Tk-owned configuration before entering the worker.
|
||
mode = self._current_mode()
|
||
under_cursor = bool(self.under_cursor_var.get())
|
||
prefer_search = bool(self.prefer_wecom_search_var.get())
|
||
start_url = (self.start_url_var.get() or "").strip()
|
||
sticky_pid = self.sticky_pid
|
||
exclude_hwnds = self._exclude_hwnds()
|
||
flow_dir = self.flow_dir
|
||
synthetic_browser_info = (
|
||
self._synthetic_browser_info()
|
||
if mode == target.SURFACE_BROWSER
|
||
else None
|
||
)
|
||
|
||
def work() -> None:
|
||
try:
|
||
uia.ensure_com_sta()
|
||
flow = art.load_flow(flow_dir) # type: ignore[arg-type]
|
||
if flow.get("status") != "open":
|
||
raise RuntimeError("流程已结束，取消本步拾取。")
|
||
if mode == target.SURFACE_ANDROID:
|
||
raise RuntimeError(
|
||
"ERROR:SURFACE_UNSUPPORTED 手机自动化尚未实现。"
|
||
)
|
||
surface = mode
|
||
sel_meta: dict = {
|
||
"pick_mode": mode,
|
||
"sticky_pid": sticky_pid,
|
||
}
|
||
if surface == target.SURFACE_BROWSER:
|
||
kit_browser.ensure_session(
|
||
start_url=start_url or "about:blank",
|
||
force_goto=False,
|
||
)
|
||
info = synthetic_browser_info
|
||
# Best-effort Win32 chrome window for meta only
|
||
try:
|
||
win_info, cands, win_meta = uia.resolve_capture_target(
|
||
exclude_hwnds=exclude_hwnds,
|
||
prefer_pid=None,
|
||
under_cursor=False,
|
||
prefer_wecom_search_popup=False,
|
||
)
|
||
if target.surface_from_window_info(win_info) == (
|
||
target.SURFACE_BROWSER
|
||
):
|
||
info = win_info
|
||
sel_meta.update(win_meta)
|
||
sel_meta["candidate_count"] = len(cands)
|
||
except Exception:
|
||
sel_meta["strategy"] = "playwright_session"
|
||
sel_meta["resolved_surface"] = surface
|
||
sel_meta["start_url"] = start_url
|
||
else:
|
||
info, cands, win_meta = uia.resolve_capture_target(
|
||
exclude_hwnds=exclude_hwnds,
|
||
prefer_pid=None if auto else sticky_pid,
|
||
under_cursor=under_cursor,
|
||
prefer_wecom_search_popup=prefer_search,
|
||
)
|
||
sel_meta.update(win_meta)
|
||
sel_meta["candidate_count"] = len(cands)
|
||
sel_meta["resolved_surface"] = surface
|
||
driver = get_driver(surface)
|
||
manual_warm_key = (
|
||
(int(info.pid), str(info.class_name or ""))
|
||
if surface == target.SURFACE_WINDOWS
|
||
else int(info.pid or 0)
|
||
)
|
||
warmup_mode = (
|
||
"light" if manual_warm_key in self._warmed_pids else "full"
|
||
)
|
||
result = driver.dump(
|
||
info=info,
|
||
selection_meta=sel_meta,
|
||
max_depth=self.max_depth,
|
||
max_nodes=self.max_nodes,
|
||
warmup_mode=warmup_mode,
|
||
warmed_pids=self._warmed_pids,
|
||
)
|
||
if info.pid:
|
||
self._warmed_pids.add(manual_warm_key)
|
||
skip_reason = skip_policy.should_skip_empty_dump(
|
||
surface=result.surface,
|
||
url=result.url,
|
||
locatable=result.locatable,
|
||
stats=result.stats,
|
||
)
|
||
if skip_reason:
|
||
msg = skip_policy.skip_reason_text(
|
||
skip_reason, url=result.url
|
||
)
|
||
self._post_ui(
|
||
self._on_pick_skipped,
|
||
msg,
|
||
auto=auto,
|
||
surface=result.surface,
|
||
url=result.url,
|
||
)
|
||
return
|
||
# Dual-write v2 checkpoint when session exists (manual)
|
||
if self._session is not None and not auto:
|
||
try:
|
||
mgr = self._session.manager
|
||
before = mgr.snapshots.latest
|
||
after = mgr.snapshots.capture_from_dump(
|
||
result, change_reason="checkpoint", force_persist=True
|
||
)
|
||
mgr._finalize_action(
|
||
action_type="checkpoint",
|
||
before=before,
|
||
after=after,
|
||
target={},
|
||
input_payload={},
|
||
raw_event={"meta": {"source": "manual_checkpoint"}},
|
||
confidence=0.9,
|
||
)
|
||
except Exception:
|
||
pass
|
||
window = result.window
|
||
locatable = result.locatable
|
||
index = result.index
|
||
stats = result.stats
|
||
label = art.resolve_step_label(
|
||
flow,
|
||
window_title=window.get("title") or info.title or "",
|
||
window_class=window.get("class_name") or info.class_name or "",
|
||
locatable=locatable,
|
||
)
|
||
step_dir, step_idx, step_name = art.add_step_dir(
|
||
self.flow_dir, # type: ignore[arg-type]
|
||
flow,
|
||
window_title=window.get("title") or info.title or "",
|
||
window_class=window.get("class_name") or info.class_name or "",
|
||
locatable=locatable,
|
||
label=label,
|
||
)
|
||
meta = art.build_meta(
|
||
window=window,
|
||
scene=flow.get("flow_name") or "flow",
|
||
surface=result.surface,
|
||
driver=result.driver,
|
||
url=result.url,
|
||
extra={
|
||
"flow_dir": str(self.flow_dir),
|
||
"step_index": step_idx,
|
||
"step_name": step_name,
|
||
"step_label": label,
|
||
"auto_pick": auto,
|
||
"dump_stats": stats,
|
||
"selection": result.selection,
|
||
},
|
||
)
|
||
art.write_json(step_dir / "meta.json", meta)
|
||
art.write_json(step_dir / "tree.json", result.tree)
|
||
art.write_jsonl(step_dir / "index.jsonl", index)
|
||
art.write_json(step_dir / "locatable.json", locatable)
|
||
png_ok = False
|
||
if result.screenshot_bytes:
|
||
try:
|
||
(step_dir / "screen.png").write_bytes(result.screenshot_bytes)
|
||
png_ok = True
|
||
except Exception:
|
||
png_ok = False
|
||
elif result.use_hwnd_screenshot:
|
||
png_ok = art.capture_window_png(info.hwnd, step_dir / "screen.png")
|
||
noteworthy = art.noteworthy_from_index(index)
|
||
art.write_step_summary(
|
||
step_dir, meta=meta, stats=stats, noteworthy=noteworthy
|
||
)
|
||
art.register_step(
|
||
self.flow_dir, # type: ignore[arg-type]
|
||
flow,
|
||
step_index=step_idx,
|
||
step_name=step_name,
|
||
window=window,
|
||
stats=stats,
|
||
label=label,
|
||
surface=result.surface,
|
||
url=result.url,
|
||
)
|
||
flow = art.load_flow(self.flow_dir) # type: ignore[arg-type]
|
||
|
||
import kit_wake
|
||
|
||
warm = stats.get("warmup") or {}
|
||
kit = stats.get("kit_wake") or {}
|
||
kit_line = kit_wake.format_kit_wake_line(kit)
|
||
sparse_note = ""
|
||
if stats.get("uia_sparse"):
|
||
sparse_note = "\n⚠ sparse=true：可定位候选偏少，请结合 screen.png。\n"
|
||
mode = "自动" if auto else "手动"
|
||
url_line = f"url: {result.url}\n" if result.url else ""
|
||
header = (
|
||
f"流程步骤: #{step_idx} `{step_name}`（{mode}）\n"
|
||
f"surface: {result.surface} ({target.surface_label(result.surface)}) "
|
||
f"driver={result.driver}\n"
|
||
f"{url_line}"
|
||
f"窗口: title={window.get('title')!r} class={window.get('class_name')!r} "
|
||
f"pid={window.get('pid')} hwnd={window.get('hwnd')}\n"
|
||
f"模式: {mode} 选窗: {sel_meta.get('strategy')}\n"
|
||
f"kit: {kit_line}\n"
|
||
f"软poke: findall {warm.get('findall_before')} → {warm.get('findall_after')} "
|
||
f"woke={warm.get('woke')}\n"
|
||
f"统计: kept={stats.get('nodes_kept')} findall={stats.get('findall_count')} "
|
||
f"locatable={stats.get('locatable_count')} png={png_ok} "
|
||
f"sparse={stats.get('uia_sparse')} cold={stats.get('provider_cold')} "
|
||
f"merge={stats.get('merge_same_pid_allowed')}\n"
|
||
f"已写入: {step_dir}\n"
|
||
f"流程文档: {self.flow_dir}\n"
|
||
f"{sparse_note}"
|
||
f"{'=' * 60}\n"
|
||
)
|
||
body = (
|
||
header
|
||
+ "\n"
|
||
+ (result.locatable_text or "")
|
||
+ "\n\n"
|
||
+ ("=" * 60)
|
||
+ "\n控件树:\n"
|
||
+ (result.tree_text or "")
|
||
)
|
||
self._post_ui(
|
||
self._on_pick_ok,
|
||
window,
|
||
flow,
|
||
body,
|
||
step_name,
|
||
auto=auto,
|
||
surface=result.surface,
|
||
)
|
||
except Exception as exc:
|
||
self._post_ui(self._on_fail, exc, auto=auto)
|
||
|
||
threading.Thread(target=work, daemon=True).start()
|
||
|
||
def _on_pick_ok(
|
||
self,
|
||
window: dict,
|
||
flow: dict,
|
||
text: str,
|
||
step_name: str,
|
||
*,
|
||
auto: bool = False,
|
||
surface: str = "windows",
|
||
) -> None:
|
||
self._busy = False
|
||
self.flow = flow
|
||
self.last_window = dict(window or {})
|
||
if window.get("pid"):
|
||
try:
|
||
self.sticky_pid = int(window["pid"])
|
||
except Exception:
|
||
pass
|
||
self._update_flow_label()
|
||
self._refresh_button_states()
|
||
url = window.get("url") or ""
|
||
url_bit = f" url={url}" if url else ""
|
||
self.target_var.set(
|
||
f"本步: surface={surface} title={window.get('title')!r} "
|
||
f"class={window.get('class_name')!r}{url_bit} | step=`{step_name}`"
|
||
)
|
||
self._set_detail(text)
|
||
if not auto:
|
||
self._set_status(
|
||
f"第 {flow.get('step_count')} 步已写入。可换页继续拾取，或点「结束流程」。"
|
||
)
|
||
try:
|
||
self.root.lift()
|
||
self.root.attributes("-topmost", True)
|
||
except Exception:
|
||
pass
|
||
if self._pending_end:
|
||
self._pending_end = False
|
||
self._finish_end_flow()
|
||
return
|
||
|
||
# auto: either pending end, or continue loop after gap
|
||
self._continue_auto_loop_after_pick()
|
||
|
||
def _on_pick_skipped(
|
||
self,
|
||
message: str,
|
||
*,
|
||
auto: bool = False,
|
||
surface: str = "browser",
|
||
url: str = "",
|
||
) -> None:
|
||
"""Blank/empty browser dump: do not write a step; keep looping if auto."""
|
||
self._busy = False
|
||
self._refresh_button_states()
|
||
self._update_flow_label()
|
||
url_bit = f" url={url}" if url else ""
|
||
self.target_var.set(f"已跳过空页（{surface}）{url_bit}")
|
||
n = int((self.flow or {}).get("step_count") or 0)
|
||
detail = (
|
||
f"{message}\n\n"
|
||
f"- 未写入 steps/（避免 about:blank / 空壳污染流程文档）\n"
|
||
f"- 当前流程已有步骤数: {n}\n"
|
||
f"- 打开目标站或等待页面加载后再拾取\n"
|
||
)
|
||
if auto:
|
||
detail += f"- 自动循环将在 {_AUTO_GAP_MS // 1000} 秒后继续\n"
|
||
self._set_detail(detail)
|
||
self._set_status(message)
|
||
if not auto:
|
||
try:
|
||
self.root.lift()
|
||
self.root.attributes("-topmost", True)
|
||
except Exception:
|
||
pass
|
||
if self._pending_end:
|
||
self._pending_end = False
|
||
self._finish_end_flow()
|
||
return
|
||
self._continue_auto_loop_after_pick()
|
||
|
||
def _on_fail(self, exc: BaseException, *, auto: bool = False) -> None:
|
||
self._busy = False
|
||
self._refresh_button_states()
|
||
self._set_status(f"失败: {exc}")
|
||
if not auto:
|
||
messagebox.showerror("UI Inspector", str(exc))
|
||
if self._pending_end:
|
||
self._pending_end = False
|
||
self._finish_end_flow()
|
||
return
|
||
# Auto: do not pop modal; keep looping after gap (unless ending)
|
||
self._continue_auto_loop_after_pick()
|
||
|
||
def _on_close(self) -> None:
|
||
if self._closed:
|
||
return
|
||
self._closed = True
|
||
self._stop_auto_loop()
|
||
self._pending_end = False
|
||
self._stop_session()
|
||
if self.flow_dir and self.flow and self.flow.get("status") == "open":
|
||
try:
|
||
latest_flow = art.load_flow(self.flow_dir)
|
||
self.flow = art.end_flow(self.flow_dir, latest_flow)
|
||
except Exception:
|
||
pass
|
||
try:
|
||
self._persist_prefs()
|
||
except Exception:
|
||
pass
|
||
try:
|
||
kit_browser.shutdown()
|
||
except Exception:
|
||
pass
|
||
try:
|
||
self.root.destroy()
|
||
except Exception:
|
||
pass
|
||
|
||
def run(self) -> None:
|
||
self.captures_root.mkdir(parents=True, exist_ok=True)
|
||
try:
|
||
self.root.mainloop()
|
||
except KeyboardInterrupt:
|
||
pass
|
||
finally:
|
||
self._on_close()
|
||
|
||
|
||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||
p = argparse.ArgumentParser(
|
||
description=(
|
||
f"OpenClaw UI Inspector v{__version__}"
|
||
"（browser/windows：动作录制、检查点与 Legacy 间隔采集）"
|
||
)
|
||
)
|
||
p.add_argument("--out", type=Path, default=None, help="输出根目录")
|
||
p.add_argument(
|
||
"--scene",
|
||
default="",
|
||
help="默认流程名（空则按拾取模式：browser / desktop / mobile）",
|
||
)
|
||
p.add_argument("--max-depth", type=int, default=16)
|
||
p.add_argument("--max-nodes", type=int, default=8000)
|
||
return p.parse_args(argv)
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
args = parse_args(argv)
|
||
out = args.out.resolve() if args.out else art.default_captures_root(_HERE)
|
||
try:
|
||
uia.ensure_uia()
|
||
except Exception as exc:
|
||
print(f"ERROR: {exc}", file=sys.stderr)
|
||
return 1
|
||
app = InspectorApp(
|
||
captures_root=out,
|
||
scene=args.scene,
|
||
max_depth=args.max_depth,
|
||
max_nodes=args.max_nodes,
|
||
)
|
||
app.run()
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|