chore: archive tools/ui_inspector snapshot before migration removal.

Keep the last in-template copy on main so history can restore it; product continues in jiangchang-ui-recorder.

Co-authored-by: Cursor <cursoragent@cursor.com>

tools/ui_inspector/recorder/__init__.py

@@ -0,0 +1,13 @@
+"""Recorder package: observe user actions outside target processes."""
+
+from .base import RawUserEvent, RecorderBase, RecorderState
+from .input_aggregator import InputAggregator
+from .manager import ActionRecordingManager
+
+__all__ = [
+    "RawUserEvent",
+    "RecorderBase",
+    "RecorderState",
+    "InputAggregator",
+    "ActionRecordingManager",
+]

tools/ui_inspector/recorder/base.py

@@ -0,0 +1,79 @@
+"""Recorder abstractions."""
+
+from __future__ import annotations
+
+import threading
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Callable, Optional
+
+
+def _now_iso() -> str:
+    return datetime.now().astimezone().isoformat()
+
+
+@dataclass
+class RawUserEvent:
+    kind: str  # mouse_down | mouse_up | key | input | navigation | focus | checkpoint
+    surface: str
+    timestamp: str = field(default_factory=_now_iso)
+    button: str = ""
+    key: str = ""
+    text: str = ""
+    x: Optional[int] = None
+    y: Optional[int] = None
+    target: dict[str, Any] = field(default_factory=dict)
+    frame: dict[str, Any] = field(default_factory=dict)
+    url: str = ""
+    meta: dict[str, Any] = field(default_factory=dict)
+    from_inspector: bool = False
+
+
+class RecorderState:
+    IDLE = "idle"
+    RECORDING = "recording"
+    PAUSED = "paused"
+    STOPPED = "stopped"
+
+
+class RecorderBase(ABC):
+    """Process-external observer. No injection / hooks into target."""
+
+    def __init__(
+        self,
+        *,
+        on_event: Optional[Callable[[RawUserEvent], None]] = None,
+        exclude_hwnds: Optional[Callable[[], set[int]]] = None,
+    ) -> None:
+        self.on_event = on_event
+        self.exclude_hwnds = exclude_hwnds or (lambda: set())
+        self.status = RecorderState.IDLE
+        self._thread: Optional[threading.Thread] = None
+        self._stop = threading.Event()
+
+    def emit(self, event: RawUserEvent) -> None:
+        if self.status != RecorderState.RECORDING:
+            return
+        if event.from_inspector:
+            return
+        if self.on_event:
+            self.on_event(event)
+
+    @abstractmethod
+    def start(self) -> None: ...
+
+    @abstractmethod
+    def stop(self) -> None: ...
+
+    def pause(self) -> None:
+        if self.status == RecorderState.RECORDING:
+            self.status = RecorderState.PAUSED
+
+    def resume(self) -> None:
+        if self.status == RecorderState.PAUSED:
+            self.status = RecorderState.RECORDING
+
+    @property
+    def capability(self) -> dict[str, str]:
+        return {}

tools/ui_inspector/recorder/browser.py

@@ -0,0 +1,740 @@
+"""Browser recorder for Inspector-owned Playwright sessions.
+
+Injects capture-phase listeners that do NOT preventDefault / stopPropagation
+and do NOT mutate business DOM. Re-attaches after navigation.
+"""
+
+from __future__ import annotations
+
+import json
+import queue
+import threading
+import time
+from typing import Any, Callable, Optional
+
+from .base import RawUserEvent, RecorderBase, RecorderState
+
+_LISTEN_SCRIPT = r"""
+(() => {
+  if (window.__openclawUiInspectorRecorderInstalled) return;
+  window.__openclawUiInspectorRecorderInstalled = true;
+  window.__openclawUiInspectorEvents = window.__openclawUiInspectorEvents || [];
+  window.__openclawUiInspectorDocId = window.__openclawUiInspectorDocId ||
+    (Date.now().toString(36) + "-" + Math.random().toString(36).slice(2));
+  window.__openclawUiInspectorSeq = window.__openclawUiInspectorSeq || 0;
+
+  function cssPath(el) {
+    if (!el || !el.tagName) return "";
+    const esc = (value) => {
+      try { return CSS.escape(String(value)); } catch (e) { return String(value).replace(/[^a-zA-Z0-9_-]/g, "\\$&"); }
+    };
+    const parts = [];
+    let cur = el;
+    for (let i = 0; cur && i < 12; i++) {
+      let part = cur.tagName.toLowerCase();
+      if (cur.id) { part += "#" + esc(cur.id); parts.unshift(part); break; }
+      const cls = (cur.className && typeof cur.className === "string")
+        ? cur.className.trim().split(/\s+/).slice(0, 3).map(esc).join(".")
+        : "";
+      if (cls) part += "." + cls;
+      parts.unshift(part);
+      cur = cur.parentElement;
+    }
+    return parts.join(" > ");
+  }
+
+  function stableCss(el) {
+    if (!el || !el.getAttribute) return "";
+    const quote = (value) => JSON.stringify(String(value));
+    const ownId = el.id || "";
+    if (ownId) return "#" + CSS.escape(ownId);
+    const testId = el.getAttribute("data-testid") || "";
+    if (testId) return `[data-testid=${quote(testId)}]`;
+    const tag = (el.tagName || "").toLowerCase();
+    let leaf = "";
+    const href = el.getAttribute("href") || "";
+    if (tag === "a" && href) leaf = `a[href=${quote(href)}]`;
+    else if (el.getAttribute("contenteditable") != null) leaf = `${tag || "*"}[contenteditable]`;
+    else if (el.getAttribute("name")) leaf = `${tag || "*"}[name=${quote(el.getAttribute("name"))}]`;
+    else if (el.getAttribute("aria-label")) leaf = `${tag || "*"}[aria-label=${quote(el.getAttribute("aria-label"))}]`;
+    else if (el.getAttribute("placeholder")) leaf = `${tag || "*"}[placeholder=${quote(el.getAttribute("placeholder"))}]`;
+    const stableClasses = (node) => {
+      if (!node || typeof node.className !== "string") return [];
+      return node.className.trim().split(/\s+/).filter((token) => {
+        if (!token || token.length < 4 || token.length > 64) return false;
+        const lower = token.toLowerCase();
+        const semantic = /(button|btn|input|editor|control|action|submit|send|publish|post|play|pause|menu|option|tab|dialog|form|toolbar|container)/.test(lower);
+        const structured = /[-_]/.test(token) && !/^[a-z0-9_-]{6,14}$/i.test(token);
+        return semantic || structured;
+      }).slice(0, 2);
+    };
+    if (!leaf) {
+      const classes = stableClasses(el);
+      leaf = tag + classes.map(cls => "." + CSS.escape(cls)).join("");
+    }
+    let cur = el.parentElement;
+    const relative = [leaf];
+    for (let i = 0; cur && i < 10; i++, cur = cur.parentElement) {
+      if (cur.id) return `#${CSS.escape(cur.id)} > ${relative.reverse().join(" > ")}`;
+      const ancestorTestId = cur.getAttribute?.("data-testid") || "";
+      if (ancestorTestId) return `[data-testid=${quote(ancestorTestId)}] > ${relative.reverse().join(" > ")}`;
+      const curTag = (cur.tagName || "").toLowerCase();
+      const classes = stableClasses(cur);
+      relative.push(curTag + classes.map(cls => "." + CSS.escape(cls)).join(""));
+    }
+    return "";
+  }
+
+  function editableValue(el) {
+    if (!el) return "";
+    if (el.value != null) return String(el.value);
+    if (el.isContentEditable || el.getAttribute?.("contenteditable") != null) {
+      return String(el.innerText != null ? el.innerText : (el.textContent || ""));
+    }
+    return "";
+  }
+
+  function isEditable(el) {
+    if (!el || !el.getAttribute) return false;
+    const tag = (el.tagName || "").toLowerCase();
+    return tag === "input" || tag === "textarea" || tag === "select" ||
+      !!el.isContentEditable || el.getAttribute("contenteditable") != null;
+  }
+
+  function editableRoot(el) {
+    let cur = el;
+    for (let i = 0; cur && i < 12; i++) {
+      if (cur.getAttribute && cur.getAttribute("contenteditable") != null) return cur;
+      const tag = (cur.tagName || "").toLowerCase();
+      if (tag === "input" || tag === "textarea" || tag === "select") return cur;
+      cur = cur.parentElement;
+    }
+    return null;
+  }
+
+  function semanticRole(el) {
+    if (!el || !el.getAttribute) return "";
+    const explicit = el.getAttribute("role");
+    if (explicit) return explicit;
+    const tag = (el.tagName || "").toLowerCase();
+    if (el.isContentEditable) return "textbox";
+    if (tag === "a") return "link";
+    if (tag === "textarea") return "textbox";
+    if (tag === "select") return "combobox";
+    if (tag === "input") {
+      const type = (el.type || "text").toLowerCase();
+      if (type === "checkbox") return "checkbox";
+      if (type === "radio") return "radio";
+      if (["button", "submit", "reset"].includes(type)) return "button";
+      return "textbox";
+    }
+    return tag;
+  }
+
+  function actionableFromEvent(ev) {
+    const path = (ev && typeof ev.composedPath === "function") ? ev.composedPath() : [];
+    const nodes = path.length ? path : [ev && ev.target];
+    let best = ev && ev.target;
+    let bestScore = -1;
+    for (const el of nodes) {
+      if (!el || !el.matches) continue;
+      const tag = (el.tagName || "").toLowerCase();
+      const role = (el.getAttribute("role") || "").toLowerCase();
+      const cls = typeof el.className === "string" ? el.className.toLowerCase() : "";
+      let cursor = "";
+      try { cursor = getComputedStyle(el).cursor || ""; } catch (e) {}
+      let score = 0;
+      if (el.matches("button,a,input,textarea,select,summary,label")) score = 100;
+      if (el.isContentEditable || el.getAttribute("contenteditable") != null) score = 98;
+      if (["button","link","textbox","combobox","checkbox","radio","menuitem","option","tab"].includes(role)) score = Math.max(score, 96);
+      else if (role) score = Math.max(score, 52);
+      if (el.getAttribute("data-testid") || el.getAttribute("aria-label")) score = Math.max(score, 90);
+      if (el.getAttribute("tabindex") != null) score = Math.max(score, 74);
+      if (cursor === "pointer") score = Math.max(score, 68);
+      // Many production apps use camelCase classes such as sendControl or
+      // playButton.  Prefer that actionable wrapper over its inherited-cursor
+      // SVG/path child while still retaining hit_target for diagnostics.
+      if (tag.includes("-") || /(button|btn|submit|send|play|control|action|click)/i.test(cls)) score = Math.max(score, 72);
+      if (score > bestScore) { best = el; bestScore = score; }
+    }
+    return best;
+  }
+
+  function snap(el) {
+    if (!el || !el.getAttribute) return {};
+    const role = semanticRole(el);
+    let cursor = "";
+    let rect = {};
+    try {
+      cursor = getComputedStyle(el).cursor || "";
+      const r = el.getBoundingClientRect();
+      rect = {left:r.left, top:r.top, right:r.right, bottom:r.bottom, width:r.width, height:r.height};
+    } catch (e) {}
+    const tag = el.tagName ? el.tagName.toLowerCase() : "";
+    const css = cssPath(el);
+    const stable = stableCss(el);
+    let cssCount = null;
+    let stableCssCount = null;
+    try { cssCount = css ? document.querySelectorAll(css).length : null; } catch (e) {}
+    try { stableCssCount = stable ? document.querySelectorAll(stable).length : null; } catch (e) {}
+    const href = el.getAttribute("href") || "";
+    let hrefCount = null;
+    try {
+      hrefCount = href ? Array.from(document.querySelectorAll("a[href]")).filter(
+        node => node.getAttribute("href") === href
+      ).length : null;
+    } catch (e) {}
+    const roleNames = ["button","link","textbox","combobox","checkbox","radio","menuitem","option","tab"];
+    return {
+      role: role,
+      explicit_role: el.getAttribute("role") || "",
+      aria_label: el.getAttribute("aria-label") || "",
+      name: el.getAttribute("aria-label") || el.name || el.id ||
+        ((el.matches?.("button,a,[role],label,summary") || roleNames.includes(role) || cursor === "pointer" || tag.includes("-"))
+          ? (el.innerText || el.textContent || "").trim().slice(0, 120) : ""),
+      text: (el.innerText || el.textContent || "").trim().slice(0, 120),
+      value: editableValue(el).slice(0, 500),
+      test_id: el.getAttribute("data-testid") || "",
+      name_attr: el.getAttribute("name") || "",
+      href: href,
+      href_count: hrefCount,
+      title: el.getAttribute("title") || "",
+      alt: el.getAttribute("alt") || "",
+      id: el.id || "",
+      placeholder: el.getAttribute("placeholder") || el.getAttribute("aria-placeholder") || "",
+      label: "",
+      input_type: (el.type || "").toLowerCase(),
+      tag: tag,
+      class_name: (typeof el.className === "string" ? el.className : "").slice(0, 300),
+      cursor: cursor,
+      rect: rect,
+      disabled: !!el.disabled || el.getAttribute("aria-disabled") === "true",
+      contenteditable: !!el.isContentEditable || el.getAttribute("contenteditable") != null,
+      contenteditable_attr: el.getAttribute("contenteditable") != null,
+      tabindex: el.getAttribute("tabindex"),
+      css: css,
+      css_count: cssCount,
+      stable_css: stable,
+      stable_css_count: stableCssCount,
+      fingerprint: [
+        el.tagName?.toLowerCase() || "", el.id || "",
+        el.getAttribute("data-testid") || "", el.getAttribute("name") || "",
+        css
+      ].join("|"),
+    };
+  }
+
+  function ancestry(el) {
+    const out = [];
+    let cur = el;
+    for (let i = 0; cur && i < 12; i++) {
+      out.push(snap(cur));
+      cur = cur.parentElement;
+    }
+    return out;
+  }
+
+  function push(kind, payload) {
+    try {
+      const item = Object.assign({
+        event_id: window.__openclawUiInspectorDocId + ":" + (++window.__openclawUiInspectorSeq),
+        document_id: window.__openclawUiInspectorDocId,
+        sequence: window.__openclawUiInspectorSeq,
+        kind: kind,
+        ts: Date.now(),
+        url: location.href,
+        title: document.title,
+        frame_url: location.href,
+        frame_name: window.name || "",
+      }, payload || {});
+      window.__openclawUiInspectorEvents.push(item);
+      if (typeof window.__openclawUiInspectorEmit === "function") {
+        try { window.__openclawUiInspectorEmit(item); } catch (e) {}
+      }
+      if (window.__openclawUiInspectorEvents.length > 200) {
+        window.__openclawUiInspectorEvents.splice(0, 50);
+      }
+    } catch (e) {}
+  }
+
+  document.addEventListener("click", (ev) => {
+    const hit = ev.target;
+    const action = actionableFromEvent(ev) || hit;
+    push("click", {
+      target: snap(action),
+      hit_target: snap(hit),
+      action_target: snap(action),
+      ancestry: ancestry(hit),
+      candidate_chain: ancestry(hit),
+      button: ev.button,
+      x: ev.clientX,
+      y: ev.clientY
+    });
+  }, true);
+
+  document.addEventListener("pointerdown", (ev) => {
+    const hit = ev.target;
+    push("pointerdown", {
+      target: snap(hit),
+      hit_target: snap(hit),
+      candidate_chain: ancestry(hit),
+      button: ev.button,
+      pointer_id: ev.pointerId,
+      pointer_type: ev.pointerType || "",
+      x: ev.clientX,
+      y: ev.clientY
+    });
+  }, true);
+
+  document.addEventListener("input", (ev) => {
+    if (ev.isComposing) return;
+    const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target;
+    push("input", {
+      target: snap(t), text: editableValue(t), input_event: ev.inputType || "input",
+      candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted
+    });
+  }, true);
+
+  document.addEventListener("change", (ev) => {
+    const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target;
+    push("input", { target: snap(t), text: editableValue(t), input_event: "change",
+      candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted });
+  }, true);
+
+  document.addEventListener("compositionend", (ev) => {
+    const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target;
+    push("input", {
+      target: snap(t), text: editableValue(t), input_event: "compositionend",
+      candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted
+    });
+  }, true);
+
+  document.addEventListener("paste", (ev) => {
+    const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target;
+    setTimeout(() => push("input", {
+      target: snap(t), text: editableValue(t), input_event: "paste",
+      candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted
+    }), 0);
+  }, true);
+
+  document.addEventListener("keydown", (ev) => {
+    const special = ["Enter","Escape","Tab","Backspace","Delete"];
+    const editing = isEditable(ev.target);
+    const mutatingEditKey = editing && (ev.key === "Backspace" || ev.key === "Delete");
+    if (!mutatingEditKey && (special.indexOf(ev.key) >= 0 || (ev.key && ev.key.length > 1))) {
+      const t = actionableFromEvent(ev) || ev.target;
+      push("key", { key: ev.key, target: snap(t) });
+    }
+  }, true);
+})();
+"""
+
+
+class BrowserRecorder(RecorderBase):
+    capability_map = {
+        "click": "implemented",
+        "fill": "implemented",
+        "press_key": "implemented",
+        "navigation": "partial",
+    }
+
+    def __init__(
+        self,
+        *,
+        on_event: Optional[Callable[[RawUserEvent], None]] = None,
+        on_raw_event: Optional[Callable[[dict[str, Any]], None]] = None,
+        poll_sec: float = 0.25,
+    ) -> None:
+        super().__init__(on_event=on_event)
+        self.on_raw_event = on_raw_event
+        self.poll_sec = poll_sec
+        self._last_url = ""
+        self._last_urls_by_page: dict[str, str] = {}
+        self._installed_pages: set[int] = set()
+        self._bound_events: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=512)
+        self._seen_event_ids: set[str] = set()
+        self._seen_event_order: list[str] = []
+        self._known_page_ids: set[str] = set()
+        self._pending_navigations: dict[str, dict[str, Any]] = {}
+        self._pending_pointerdowns: dict[str, dict[str, Any]] = {}
+        self._navigation_reorder_sec = 0.45
+        self._pointer_fallback_sec = 0.35
+
+    @property
+    def capability(self) -> dict[str, str]:
+        return dict(self.capability_map)
+
+    def start(self) -> None:
+        if self._thread and self._thread.is_alive():
+            self.status = RecorderState.RECORDING
+            return
+        self._stop.clear()
+        self.status = RecorderState.RECORDING
+        try:
+            import kit_browser
+
+            kit_browser.install_event_bridge(_LISTEN_SCRIPT, self._receive_bound_event)
+            initial_ids = kit_browser.call_on_each_open_page(
+                lambda page: kit_browser.page_id_for(page)
+            )
+            self._known_page_ids.update(
+                str(page_id) for page_id in initial_ids if page_id
+            )
+        except Exception:
+            # Polling remains as a compatibility fallback.
+            pass
+        self._thread = threading.Thread(
+            target=self._loop, name="ui-inspector-browser-recorder", daemon=True
+        )
+        self._thread.start()
+
+    def stop(self) -> None:
+        self._stop.set()
+        if self._thread and self._thread.is_alive():
+            self._thread.join(timeout=2.0)
+        self._thread = None
+        # Disable the bridge before draining its local queue so an event that
+        # arrived just before Stop is not silently lost, while no new events
+        # can race with the drain.
+        try:
+            import kit_browser
+
+            kit_browser.clear_event_bridge()
+        except Exception:
+            pass
+        if self.status == RecorderState.RECORDING:
+            try:
+                self._drain_bound_events()
+                self._flush_pending_pointerdowns(force=True)
+                self._flush_pending_navigations(force=True)
+            except Exception:
+                pass
+        self.status = RecorderState.STOPPED
+
+    def _receive_bound_event(self, event: dict[str, Any]) -> None:
+        try:
+            self._bound_events.put_nowait(dict(event or {}))
+        except queue.Full:
+            try:
+                self._bound_events.get_nowait()
+                self._bound_events.put_nowait(dict(event or {}))
+            except (queue.Empty, queue.Full):
+                pass
+
+    def _loop(self) -> None:
+        while not self._stop.is_set():
+            if self.status == RecorderState.PAUSED:
+                time.sleep(0.1)
+                continue
+            if self.status != RecorderState.RECORDING:
+                time.sleep(0.05)
+                continue
+            try:
+                self._tick()
+            except Exception:
+                pass
+            time.sleep(self.poll_sec)
+
+    def _tick(self) -> None:
+        import kit_browser
+
+        self._drain_bound_events()
+
+        # Install + drain on every open tab (detail may be a second tab).
+        payloads = kit_browser.call_on_each_open_page(self._install_and_drain)
+        if payloads:
+            for payload in payloads:
+                if not isinstance(payload, dict):
+                    continue
+                page_id = str(payload.get("page_id") or "")
+                url = str(payload.get("url") or "")
+                previous_url = self._last_urls_by_page.get(page_id, "") if page_id else ""
+                if previous_url and url and url != previous_url:
+                    self._schedule_navigation(
+                        page_id=page_id, from_url=previous_url, to_url=url
+                    )
+                elif (
+                    page_id
+                    and page_id not in self._known_page_ids
+                    and self._known_page_ids
+                    and url
+                    and not url.startswith("about:")
+                ):
+                    opener_id = str(payload.get("opener_page_id") or "")
+                    self._schedule_navigation(
+                        page_id=page_id,
+                        from_url=self._last_urls_by_page.get(opener_id, ""),
+                        to_url=url,
+                        opener_page_id=opener_id,
+                        new_page=True,
+                    )
+                if page_id and url:
+                    self._last_urls_by_page[page_id] = url
+                    self._known_page_ids.add(page_id)
+                for ev in payload.get("events") or []:
+                    if isinstance(ev, dict) and not ev.get("page_id"):
+                        ev["page_id"] = payload.get("page_id") or ""
+                    self._emit_dom_event(ev)
+
+        self._flush_pending_pointerdowns()
+
+        # Navigation is intentionally emitted after both bridge and polling
+        # queues have been drained.  This preserves click -> navigation order
+        # even when the context bridge is unavailable and polling is used.
+        self._flush_pending_navigations()
+
+    def _drain_bound_events(self) -> None:
+        while True:
+            try:
+                self._emit_dom_event(self._bound_events.get_nowait())
+            except queue.Empty:
+                break
+
+    def _schedule_navigation(
+        self,
+        *,
+        page_id: str,
+        from_url: str,
+        to_url: str,
+        opener_page_id: str = "",
+        new_page: bool = False,
+    ) -> None:
+        self._pending_navigations[page_id] = {
+            "due": time.monotonic() + self._navigation_reorder_sec,
+            "from": from_url,
+            "to": to_url,
+            "page_id": page_id,
+            "opener_page_id": opener_page_id,
+            "new_page": new_page,
+        }
+
+    def _flush_pending_navigations(self, *, force: bool = False) -> None:
+        now = time.monotonic()
+        ready = [
+            page_id
+            for page_id, item in self._pending_navigations.items()
+            if force or float(item.get("due") or 0.0) <= now
+        ]
+        for page_id in ready:
+            self._flush_pending_pointerdowns(force=True, page_id=page_id)
+            item = self._pending_navigations.pop(page_id, None) or {}
+            self.emit(
+                RawUserEvent(
+                    kind="navigation",
+                    surface="browser",
+                    url=str(item.get("to") or ""),
+                    meta={
+                        "from": item.get("from") or "",
+                        "to": item.get("to") or "",
+                        "page_id": page_id,
+                        "opener_page_id": item.get("opener_page_id") or "",
+                        "new_page": bool(item.get("new_page")),
+                    },
+                )
+            )
+
+    def _install_and_drain(self, page: Any) -> dict[str, Any]:
+        """Runs on Playwright worker thread (once per open tab)."""
+        try:
+            page.evaluate(_LISTEN_SCRIPT)
+        except Exception:
+            pass
+        # Also try frames
+        try:
+            for frame in page.frames:
+                try:
+                    frame.evaluate(_LISTEN_SCRIPT)
+                except Exception:
+                    pass
+        except Exception:
+            pass
+        events = []
+        try:
+            events = page.evaluate(
+                """() => {
+                  const e = window.__openclawUiInspectorEvents || [];
+                  window.__openclawUiInspectorEvents = [];
+                  return e;
+                }"""
+            ) or []
+        except Exception:
+            events = []
+        url = ""
+        try:
+            url = page.url or ""
+        except Exception:
+            url = ""
+        page_id = ""
+        opener_page_id = ""
+        try:
+            import kit_browser
+
+            page_id = kit_browser.page_id_for(page)
+            try:
+                opener = page.opener()
+                if opener is not None:
+                    opener_page_id = kit_browser.page_id_for(opener)
+            except Exception:
+                pass
+        except Exception:
+            pass
+        return {
+            "url": url,
+            "page_id": page_id,
+            "opener_page_id": opener_page_id,
+            "events": events,
+        }
+
+    def _emit_dom_event(self, ev: dict[str, Any]) -> None:
+        event_id = str(ev.get("event_id") or "")
+        if event_id:
+            if event_id in self._seen_event_ids:
+                return
+            self._seen_event_ids.add(event_id)
+            self._seen_event_order.append(event_id)
+            if len(self._seen_event_order) > 2000:
+                old = self._seen_event_order.pop(0)
+                self._seen_event_ids.discard(old)
+        if self.on_raw_event:
+            try:
+                self.on_raw_event(dict(ev))
+            except Exception:
+                pass
+        kind = str(ev.get("kind") or "")
+        hit_target = dict(ev.get("hit_target") or ev.get("target") or {})
+        candidate_chain = list(ev.get("candidate_chain") or ev.get("ancestry") or [])
+        if not candidate_chain:
+            candidate_chain = [hit_target]
+            action_hint = dict(ev.get("action_target") or {})
+            if action_hint and action_hint != hit_target:
+                candidate_chain.append(action_hint)
+        try:
+            from core.policies.target_scoring import resolve_browser_target
+
+            target, target_scores = resolve_browser_target(
+                hit_target,
+                candidate_chain,
+                x=int(ev.get("x")) if ev.get("x") is not None else None,
+                y=int(ev.get("y")) if ev.get("y") is not None else None,
+            )
+        except Exception:
+            target = dict(ev.get("action_target") or ev.get("target") or hit_target)
+            target_scores = []
+        if candidate_chain:
+            target["parent_chain"] = candidate_chain
+        frame = {
+            "url": ev.get("frame_url") or "",
+            "name": ev.get("frame_name") or "",
+            "id": ev.get("frame_id") or "",
+        }
+        if kind in {"pointerdown", "click"}:
+            raw = RawUserEvent(
+                    kind="click",
+                    surface="browser",
+                    target=target,
+                    frame=frame,
+                    url=str(ev.get("url") or ""),
+                    x=int(ev.get("x")) if ev.get("x") is not None else None,
+                    y=int(ev.get("y")) if ev.get("y") is not None else None,
+                    meta={
+                        "button": ev.get("button"),
+                        "event_id": event_id,
+                        "document_id": ev.get("document_id"),
+                        "page_id": ev.get("page_id"),
+                        "frame_id": ev.get("frame_id"),
+                        "sequence": ev.get("sequence"),
+                        "source_ts_ms": ev.get("ts"),
+                        "hit_target": ev.get("hit_target") or {},
+                        "action_target": ev.get("action_target") or target,
+                        "target_scores": target_scores,
+                        "captured_from": kind,
+                    },
+                )
+            if kind == "pointerdown":
+                key = self._pointer_key(ev)
+                self._pending_pointerdowns[key] = {
+                    "due": time.monotonic() + self._pointer_fallback_sec,
+                    "event": raw,
+                    "page_id": str(ev.get("page_id") or ""),
+                }
+                return
+            self._cancel_pointerdown_for_click(ev)
+            self.emit(raw)
+        elif kind == "input":
+            self.emit(
+                RawUserEvent(
+                    kind="input",
+                    surface="browser",
+                    text=str(ev.get("text") or target.get("value") or ""),
+                    target=target,
+                    frame=frame,
+                    url=str(ev.get("url") or ""),
+                    meta={
+                        "input_type": target.get("input_type") or "",
+                        "event_id": event_id,
+                        "document_id": ev.get("document_id"),
+                        "page_id": ev.get("page_id"),
+                        "frame_id": ev.get("frame_id"),
+                        "sequence": ev.get("sequence"),
+                        "source_ts_ms": ev.get("ts"),
+                        "input_event": ev.get("input_event") or "input",
+                        "is_trusted": bool(ev.get("is_trusted")),
+                    },
+                )
+            )
+        elif kind == "key":
+            self.emit(
+                RawUserEvent(
+                    kind="key",
+                    surface="browser",
+                    key=str(ev.get("key") or ""),
+                    target=target,
+                    frame=frame,
+                    url=str(ev.get("url") or ""),
+                    meta={
+                        "event_id": event_id,
+                        "document_id": ev.get("document_id"),
+                        "page_id": ev.get("page_id"),
+                        "frame_id": ev.get("frame_id"),
+                        "sequence": ev.get("sequence"),
+                        "source_ts_ms": ev.get("ts"),
+                    },
+                )
+            )
+
+    @staticmethod
+    def _pointer_key(ev: dict[str, Any]) -> str:
+        return "|".join(
+            str(ev.get(key) or "")
+            for key in ("page_id", "document_id", "pointer_id", "button")
+        )
+
+    def _cancel_pointerdown_for_click(self, ev: dict[str, Any]) -> None:
+        exact = self._pointer_key(ev)
+        if exact in self._pending_pointerdowns:
+            self._pending_pointerdowns.pop(exact, None)
+            return
+        page_id = str(ev.get("page_id") or "")
+        button = str(ev.get("button") or "")
+        for key, item in list(self._pending_pointerdowns.items()):
+            event = item.get("event")
+            meta = getattr(event, "meta", {}) or {}
+            if str(meta.get("page_id") or "") == page_id and str(meta.get("button") or "") == button:
+                self._pending_pointerdowns.pop(key, None)
+                return
+
+    def _flush_pending_pointerdowns(
+        self, *, force: bool = False, page_id: str = ""
+    ) -> None:
+        now = time.monotonic()
+        ready = [
+            key for key, item in self._pending_pointerdowns.items()
+            if (not page_id or item.get("page_id") == page_id)
+            and (force or float(item.get("due") or 0.0) <= now)
+        ]
+        for key in ready:
+            item = self._pending_pointerdowns.pop(key, None) or {}
+            event = item.get("event")
+            if event is not None:
+                event.meta["pointerdown_fallback"] = True
+                self.emit(event)

tools/ui_inspector/recorder/input_aggregator.py

@@ -0,0 +1,234 @@
+"""Aggregate keystrokes / value changes into fill/type + press_key."""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+from redaction import redact_input_payload
+
+# Keys recorded as independent press_key actions.
+_FUNCTION_KEYS = {
+    "Enter",
+    "Escape",
+    "Tab",
+    "Backspace",
+    "Delete",
+    "ArrowUp",
+    "ArrowDown",
+    "ArrowLeft",
+    "ArrowRight",
+    "Home",
+    "End",
+    "PageUp",
+    "PageDown",
+    "F1",
+    "F2",
+    "F3",
+    "F4",
+    "F5",
+    "F6",
+    "F7",
+    "F8",
+    "F9",
+    "F10",
+    "F11",
+    "F12",
+}
+
+
+@dataclass
+class AggregatedInput:
+    action_type: str  # fill | type | press_key
+    text: str = ""
+    key: str = ""
+    target: dict[str, Any] = field(default_factory=dict)
+    sensitive: bool = False
+    input_payload: dict[str, Any] = field(default_factory=dict)
+    started_at: float = 0.0
+    ended_at: float = 0.0
+    capture_context: dict[str, Any] = field(default_factory=dict)
+    raw_event: dict[str, Any] = field(default_factory=dict)
+    raw_event_ids: list[str] = field(default_factory=list)
+
+
+class InputAggregator:
+    """
+    Do not emit one Action per key.
+    Observe focus element value changes; flush after idle gap or function key.
+    """
+
+    def __init__(self, *, idle_sec: float = 0.8) -> None:
+        self.idle_sec = idle_sec
+        self._buf_text = ""
+        self._target: dict[str, Any] = {}
+        self._started: float = 0.0
+        self._last: float = 0.0
+        self._input_type = ""
+        self._field_name = ""
+        self._capture_context: dict[str, Any] = {}
+        self._raw_event: dict[str, Any] = {}
+        self._raw_event_ids: list[str] = []
+        self._target_identity = ""
+
+    @staticmethod
+    def _stable_target_identity(
+        target: dict[str, Any], capture_context: dict[str, Any]
+    ) -> str:
+        from core.policies.element_identity import element_identity_key
+
+        return element_identity_key(
+            target or {},
+            surface=str(capture_context.get("surface") or "windows"),
+            page_url=str(capture_context.get("url") or ""),
+            frame_name=str(capture_context.get("frame_name") or ""),
+        )
+
+    @staticmethod
+    def _event_id(raw_event: Optional[dict[str, Any]]) -> str:
+        return str(((raw_event or {}).get("meta") or {}).get("event_id") or "")
+
+    def on_key(
+        self,
+        *,
+        key: str,
+        text: str = "",
+        target: Optional[dict[str, Any]] = None,
+        input_type: str = "",
+        field_name: str = "",
+        capture_context: Optional[dict[str, Any]] = None,
+        raw_event: Optional[dict[str, Any]] = None,
+        value_event: bool = False,
+        now: Optional[float] = None,
+    ) -> list[AggregatedInput]:
+        now = time.monotonic() if now is None else now
+        out: list[AggregatedInput] = []
+        k = key or ""
+        if k in _FUNCTION_KEYS or (len(k) > 1 and k.startswith("F") and k[1:].isdigit()):
+            out.extend(self.flush(now=now))
+            payload, sens = redact_input_payload(
+                {"key": k},
+                field_name=field_name or self._field_name,
+                input_type=input_type or self._input_type,
+            )
+            out.append(
+                AggregatedInput(
+                    action_type="press_key",
+                    key=k,
+                    target=dict(target or self._target or {}),
+                    sensitive=sens,
+                    input_payload=payload,
+                    started_at=now,
+                    ended_at=now,
+                    capture_context=dict(capture_context or self._capture_context or {}),
+                    raw_event=dict(raw_event or {}),
+                    raw_event_ids=(
+                        [self._event_id(raw_event)] if self._event_id(raw_event) else []
+                    ),
+                )
+            )
+            return out
+
+        # Printable / value change
+        if target:
+            # Focus changed → flush previous
+            next_context = dict(capture_context or self._capture_context or {})
+            next_identity = self._stable_target_identity(target, next_context)
+            if self._target_identity and next_identity != self._target_identity:
+                out.extend(self.flush(now=now))
+            self._target = dict(target)
+            self._target_identity = next_identity
+        if input_type:
+            self._input_type = input_type
+        if field_name:
+            self._field_name = field_name
+        if capture_context:
+            self._capture_context = dict(capture_context)
+        if raw_event:
+            self._raw_event = dict(raw_event)
+            event_id = self._event_id(raw_event)
+            if event_id and event_id not in self._raw_event_ids:
+                self._raw_event_ids.append(event_id)
+
+        chunk = text if text else (k if len(k) == 1 else "")
+        if not chunk and not text and not value_event:
+            return out
+
+        if not self._started:
+            self._started = now
+        # If observer sends full value, replace buffer; else append char
+        if value_event:
+            self._buf_text = text
+        elif text and len(text) >= len(self._buf_text):
+            self._buf_text = text
+        else:
+            self._buf_text += chunk
+        self._last = now
+        return out
+
+    def on_value_change(
+        self,
+        *,
+        value: str,
+        target: Optional[dict[str, Any]] = None,
+        input_type: str = "",
+        field_name: str = "",
+        capture_context: Optional[dict[str, Any]] = None,
+        raw_event: Optional[dict[str, Any]] = None,
+        now: Optional[float] = None,
+    ) -> list[AggregatedInput]:
+        return self.on_key(
+            key="",
+            text=value,
+            target=target,
+            input_type=input_type,
+            field_name=field_name,
+            capture_context=capture_context,
+            raw_event=raw_event,
+            value_event=True,
+            now=now,
+        )
+
+    def flush(self, *, now: Optional[float] = None) -> list[AggregatedInput]:
+        now = (
+            (time.time() if self._last > 100_000_000 else time.monotonic())
+            if now is None else now
+        )
+        if not self._buf_text and not self._started:
+            return []
+        payload, sens = redact_input_payload(
+            {"text": self._buf_text, "name": self._field_name},
+            field_name=self._field_name,
+            input_type=self._input_type,
+        )
+        item = AggregatedInput(
+            action_type="fill",
+            text=str(payload.get("text") or ""),
+            target=dict(self._target or {}),
+            sensitive=sens,
+            input_payload=payload,
+            started_at=self._started or now,
+            ended_at=now,
+            capture_context=dict(self._capture_context or {}),
+            raw_event=dict(getattr(self, "_raw_event", {}) or {}),
+            raw_event_ids=list(self._raw_event_ids),
+        )
+        self._buf_text = ""
+        self._started = 0.0
+        self._last = 0.0
+        self._input_type = ""
+        self._capture_context = {}
+        self._raw_event = {}
+        self._raw_event_ids = []
+        # keep target for next session
+        return [item]
+
+    def maybe_flush_idle(self, *, now: Optional[float] = None) -> list[AggregatedInput]:
+        now = (
+            (time.time() if self._last > 100_000_000 else time.monotonic())
+            if now is None else now
+        )
+        if self._last and (now - self._last) >= self.idle_sec and self._started:
+            return self.flush(now=now)
+        return []