"""Browser recorder for Inspector-owned Playwright sessions. Injects capture-phase listeners that do NOT preventDefault / stopPropagation and do NOT mutate business DOM. Re-attaches after navigation. """ from __future__ import annotations import json import queue import threading import time from typing import Any, Callable, Optional from .base import RawUserEvent, RecorderBase, RecorderState _LISTEN_SCRIPT = r""" (() => { if (window.__openclawUiInspectorRecorderInstalled) return; window.__openclawUiInspectorRecorderInstalled = true; window.__openclawUiInspectorEvents = window.__openclawUiInspectorEvents || []; window.__openclawUiInspectorDocId = window.__openclawUiInspectorDocId || (Date.now().toString(36) + "-" + Math.random().toString(36).slice(2)); window.__openclawUiInspectorSeq = window.__openclawUiInspectorSeq || 0; function cssPath(el) { if (!el || !el.tagName) return ""; const esc = (value) => { try { return CSS.escape(String(value)); } catch (e) { return String(value).replace(/[^a-zA-Z0-9_-]/g, "\\$&"); } }; const parts = []; let cur = el; for (let i = 0; cur && i < 12; i++) { let part = cur.tagName.toLowerCase(); if (cur.id) { part += "#" + esc(cur.id); parts.unshift(part); break; } const cls = (cur.className && typeof cur.className === "string") ? cur.className.trim().split(/\s+/).slice(0, 3).map(esc).join(".") : ""; if (cls) part += "." + cls; parts.unshift(part); cur = cur.parentElement; } return parts.join(" > "); } function stableCss(el) { if (!el || !el.getAttribute) return ""; const quote = (value) => JSON.stringify(String(value)); const ownId = el.id || ""; if (ownId) return "#" + CSS.escape(ownId); const testId = el.getAttribute("data-testid") || ""; if (testId) return `[data-testid=${quote(testId)}]`; const tag = (el.tagName || "").toLowerCase(); let leaf = ""; const href = el.getAttribute("href") || ""; if (tag === "a" && href) leaf = `a[href=${quote(href)}]`; else if (el.getAttribute("contenteditable") != null) leaf = `${tag || "*"}[contenteditable]`; else if (el.getAttribute("name")) leaf = `${tag || "*"}[name=${quote(el.getAttribute("name"))}]`; else if (el.getAttribute("aria-label")) leaf = `${tag || "*"}[aria-label=${quote(el.getAttribute("aria-label"))}]`; else if (el.getAttribute("placeholder")) leaf = `${tag || "*"}[placeholder=${quote(el.getAttribute("placeholder"))}]`; const stableClasses = (node) => { if (!node || typeof node.className !== "string") return []; return node.className.trim().split(/\s+/).filter((token) => { if (!token || token.length < 4 || token.length > 64) return false; const lower = token.toLowerCase(); const semantic = /(button|btn|input|editor|control|action|submit|send|publish|post|play|pause|menu|option|tab|dialog|form|toolbar|container)/.test(lower); const structured = /[-_]/.test(token) && !/^[a-z0-9_-]{6,14}$/i.test(token); return semantic || structured; }).slice(0, 2); }; if (!leaf) { const classes = stableClasses(el); leaf = tag + classes.map(cls => "." + CSS.escape(cls)).join(""); } let cur = el.parentElement; const relative = [leaf]; for (let i = 0; cur && i < 10; i++, cur = cur.parentElement) { if (cur.id) return `#${CSS.escape(cur.id)} > ${relative.reverse().join(" > ")}`; const ancestorTestId = cur.getAttribute?.("data-testid") || ""; if (ancestorTestId) return `[data-testid=${quote(ancestorTestId)}] > ${relative.reverse().join(" > ")}`; const curTag = (cur.tagName || "").toLowerCase(); const classes = stableClasses(cur); relative.push(curTag + classes.map(cls => "." + CSS.escape(cls)).join("")); } return ""; } function editableValue(el) { if (!el) return ""; if (el.value != null) return String(el.value); if (el.isContentEditable || el.getAttribute?.("contenteditable") != null) { return String(el.innerText != null ? el.innerText : (el.textContent || "")); } return ""; } function isEditable(el) { if (!el || !el.getAttribute) return false; const tag = (el.tagName || "").toLowerCase(); return tag === "input" || tag === "textarea" || tag === "select" || !!el.isContentEditable || el.getAttribute("contenteditable") != null; } function editableRoot(el) { let cur = el; for (let i = 0; cur && i < 12; i++) { if (cur.getAttribute && cur.getAttribute("contenteditable") != null) return cur; const tag = (cur.tagName || "").toLowerCase(); if (tag === "input" || tag === "textarea" || tag === "select") return cur; cur = cur.parentElement; } return null; } function semanticRole(el) { if (!el || !el.getAttribute) return ""; const explicit = el.getAttribute("role"); if (explicit) return explicit; const tag = (el.tagName || "").toLowerCase(); if (el.isContentEditable) return "textbox"; if (tag === "a") return "link"; if (tag === "textarea") return "textbox"; if (tag === "select") return "combobox"; if (tag === "input") { const type = (el.type || "text").toLowerCase(); if (type === "checkbox") return "checkbox"; if (type === "radio") return "radio"; if (["button", "submit", "reset"].includes(type)) return "button"; return "textbox"; } return tag; } function actionableFromEvent(ev) { const path = (ev && typeof ev.composedPath === "function") ? ev.composedPath() : []; const nodes = path.length ? path : [ev && ev.target]; let best = ev && ev.target; let bestScore = -1; for (const el of nodes) { if (!el || !el.matches) continue; const tag = (el.tagName || "").toLowerCase(); const role = (el.getAttribute("role") || "").toLowerCase(); const cls = typeof el.className === "string" ? el.className.toLowerCase() : ""; let cursor = ""; try { cursor = getComputedStyle(el).cursor || ""; } catch (e) {} let score = 0; if (el.matches("button,a,input,textarea,select,summary,label")) score = 100; if (el.isContentEditable || el.getAttribute("contenteditable") != null) score = 98; if (["button","link","textbox","combobox","checkbox","radio","menuitem","option","tab"].includes(role)) score = Math.max(score, 96); else if (role) score = Math.max(score, 52); if (el.getAttribute("data-testid") || el.getAttribute("aria-label")) score = Math.max(score, 90); if (el.getAttribute("tabindex") != null) score = Math.max(score, 74); if (cursor === "pointer") score = Math.max(score, 68); // Many production apps use camelCase classes such as sendControl or // playButton. Prefer that actionable wrapper over its inherited-cursor // SVG/path child while still retaining hit_target for diagnostics. if (tag.includes("-") || /(button|btn|submit|send|play|control|action|click)/i.test(cls)) score = Math.max(score, 72); if (score > bestScore) { best = el; bestScore = score; } } return best; } function snap(el) { if (!el || !el.getAttribute) return {}; const role = semanticRole(el); let cursor = ""; let rect = {}; try { cursor = getComputedStyle(el).cursor || ""; const r = el.getBoundingClientRect(); rect = {left:r.left, top:r.top, right:r.right, bottom:r.bottom, width:r.width, height:r.height}; } catch (e) {} const tag = el.tagName ? el.tagName.toLowerCase() : ""; const css = cssPath(el); const stable = stableCss(el); let cssCount = null; let stableCssCount = null; try { cssCount = css ? document.querySelectorAll(css).length : null; } catch (e) {} try { stableCssCount = stable ? document.querySelectorAll(stable).length : null; } catch (e) {} const href = el.getAttribute("href") || ""; let hrefCount = null; try { hrefCount = href ? Array.from(document.querySelectorAll("a[href]")).filter( node => node.getAttribute("href") === href ).length : null; } catch (e) {} const roleNames = ["button","link","textbox","combobox","checkbox","radio","menuitem","option","tab"]; return { role: role, explicit_role: el.getAttribute("role") || "", aria_label: el.getAttribute("aria-label") || "", name: el.getAttribute("aria-label") || el.name || el.id || ((el.matches?.("button,a,[role],label,summary") || roleNames.includes(role) || cursor === "pointer" || tag.includes("-")) ? (el.innerText || el.textContent || "").trim().slice(0, 120) : ""), text: (el.innerText || el.textContent || "").trim().slice(0, 120), value: editableValue(el).slice(0, 500), test_id: el.getAttribute("data-testid") || "", name_attr: el.getAttribute("name") || "", href: href, href_count: hrefCount, title: el.getAttribute("title") || "", alt: el.getAttribute("alt") || "", id: el.id || "", placeholder: el.getAttribute("placeholder") || el.getAttribute("aria-placeholder") || "", label: "", input_type: (el.type || "").toLowerCase(), tag: tag, class_name: (typeof el.className === "string" ? el.className : "").slice(0, 300), cursor: cursor, rect: rect, disabled: !!el.disabled || el.getAttribute("aria-disabled") === "true", contenteditable: !!el.isContentEditable || el.getAttribute("contenteditable") != null, contenteditable_attr: el.getAttribute("contenteditable") != null, tabindex: el.getAttribute("tabindex"), css: css, css_count: cssCount, stable_css: stable, stable_css_count: stableCssCount, fingerprint: [ el.tagName?.toLowerCase() || "", el.id || "", el.getAttribute("data-testid") || "", el.getAttribute("name") || "", css ].join("|"), }; } function ancestry(el) { const out = []; let cur = el; for (let i = 0; cur && i < 12; i++) { out.push(snap(cur)); cur = cur.parentElement; } return out; } function push(kind, payload) { try { const item = Object.assign({ event_id: window.__openclawUiInspectorDocId + ":" + (++window.__openclawUiInspectorSeq), document_id: window.__openclawUiInspectorDocId, sequence: window.__openclawUiInspectorSeq, kind: kind, ts: Date.now(), url: location.href, title: document.title, frame_url: location.href, frame_name: window.name || "", }, payload || {}); window.__openclawUiInspectorEvents.push(item); if (typeof window.__openclawUiInspectorEmit === "function") { try { window.__openclawUiInspectorEmit(item); } catch (e) {} } if (window.__openclawUiInspectorEvents.length > 200) { window.__openclawUiInspectorEvents.splice(0, 50); } } catch (e) {} } document.addEventListener("click", (ev) => { const hit = ev.target; const action = actionableFromEvent(ev) || hit; push("click", { target: snap(action), hit_target: snap(hit), action_target: snap(action), ancestry: ancestry(hit), candidate_chain: ancestry(hit), button: ev.button, x: ev.clientX, y: ev.clientY }); }, true); document.addEventListener("pointerdown", (ev) => { const hit = ev.target; push("pointerdown", { target: snap(hit), hit_target: snap(hit), candidate_chain: ancestry(hit), button: ev.button, pointer_id: ev.pointerId, pointer_type: ev.pointerType || "", x: ev.clientX, y: ev.clientY }); }, true); document.addEventListener("input", (ev) => { if (ev.isComposing) return; const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target; push("input", { target: snap(t), text: editableValue(t), input_event: ev.inputType || "input", candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted }); }, true); document.addEventListener("change", (ev) => { const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target; push("input", { target: snap(t), text: editableValue(t), input_event: "change", candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted }); }, true); document.addEventListener("compositionend", (ev) => { const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target; push("input", { target: snap(t), text: editableValue(t), input_event: "compositionend", candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted }); }, true); document.addEventListener("paste", (ev) => { const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target; setTimeout(() => push("input", { target: snap(t), text: editableValue(t), input_event: "paste", candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted }), 0); }, true); document.addEventListener("keydown", (ev) => { const special = ["Enter","Escape","Tab","Backspace","Delete"]; const editing = isEditable(ev.target); const mutatingEditKey = editing && (ev.key === "Backspace" || ev.key === "Delete"); if (!mutatingEditKey && (special.indexOf(ev.key) >= 0 || (ev.key && ev.key.length > 1))) { const t = actionableFromEvent(ev) || ev.target; push("key", { key: ev.key, target: snap(t) }); } }, true); })(); """ class BrowserRecorder(RecorderBase): capability_map = { "click": "implemented", "fill": "implemented", "press_key": "implemented", "navigation": "partial", } def __init__( self, *, on_event: Optional[Callable[[RawUserEvent], None]] = None, on_raw_event: Optional[Callable[[dict[str, Any]], None]] = None, poll_sec: float = 0.25, ) -> None: super().__init__(on_event=on_event) self.on_raw_event = on_raw_event self.poll_sec = poll_sec self._last_url = "" self._last_urls_by_page: dict[str, str] = {} self._installed_pages: set[int] = set() self._bound_events: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=512) self._seen_event_ids: set[str] = set() self._seen_event_order: list[str] = [] self._known_page_ids: set[str] = set() self._pending_navigations: dict[str, dict[str, Any]] = {} self._pending_pointerdowns: dict[str, dict[str, Any]] = {} self._navigation_reorder_sec = 0.45 self._pointer_fallback_sec = 0.35 @property def capability(self) -> dict[str, str]: return dict(self.capability_map) def start(self) -> None: if self._thread and self._thread.is_alive(): self.status = RecorderState.RECORDING return self._stop.clear() self.status = RecorderState.RECORDING try: import kit_browser kit_browser.install_event_bridge(_LISTEN_SCRIPT, self._receive_bound_event) initial_ids = kit_browser.call_on_each_open_page( lambda page: kit_browser.page_id_for(page) ) self._known_page_ids.update( str(page_id) for page_id in initial_ids if page_id ) except Exception: # Polling remains as a compatibility fallback. pass self._thread = threading.Thread( target=self._loop, name="ui-inspector-browser-recorder", daemon=True ) self._thread.start() def stop(self) -> None: self._stop.set() if self._thread and self._thread.is_alive(): self._thread.join(timeout=2.0) self._thread = None # Disable the bridge before draining its local queue so an event that # arrived just before Stop is not silently lost, while no new events # can race with the drain. try: import kit_browser kit_browser.clear_event_bridge() except Exception: pass if self.status == RecorderState.RECORDING: try: self._drain_bound_events() self._flush_pending_pointerdowns(force=True) self._flush_pending_navigations(force=True) except Exception: pass self.status = RecorderState.STOPPED def _receive_bound_event(self, event: dict[str, Any]) -> None: try: self._bound_events.put_nowait(dict(event or {})) except queue.Full: try: self._bound_events.get_nowait() self._bound_events.put_nowait(dict(event or {})) except (queue.Empty, queue.Full): pass def _loop(self) -> None: while not self._stop.is_set(): if self.status == RecorderState.PAUSED: time.sleep(0.1) continue if self.status != RecorderState.RECORDING: time.sleep(0.05) continue try: self._tick() except Exception: pass time.sleep(self.poll_sec) def _tick(self) -> None: import kit_browser self._drain_bound_events() # Install + drain on every open tab (detail may be a second tab). payloads = kit_browser.call_on_each_open_page(self._install_and_drain) if payloads: for payload in payloads: if not isinstance(payload, dict): continue page_id = str(payload.get("page_id") or "") url = str(payload.get("url") or "") previous_url = self._last_urls_by_page.get(page_id, "") if page_id else "" if previous_url and url and url != previous_url: self._schedule_navigation( page_id=page_id, from_url=previous_url, to_url=url ) elif ( page_id and page_id not in self._known_page_ids and self._known_page_ids and url and not url.startswith("about:") ): opener_id = str(payload.get("opener_page_id") or "") self._schedule_navigation( page_id=page_id, from_url=self._last_urls_by_page.get(opener_id, ""), to_url=url, opener_page_id=opener_id, new_page=True, ) if page_id and url: self._last_urls_by_page[page_id] = url self._known_page_ids.add(page_id) for ev in payload.get("events") or []: if isinstance(ev, dict) and not ev.get("page_id"): ev["page_id"] = payload.get("page_id") or "" self._emit_dom_event(ev) self._flush_pending_pointerdowns() # Navigation is intentionally emitted after both bridge and polling # queues have been drained. This preserves click -> navigation order # even when the context bridge is unavailable and polling is used. self._flush_pending_navigations() def _drain_bound_events(self) -> None: while True: try: self._emit_dom_event(self._bound_events.get_nowait()) except queue.Empty: break def _schedule_navigation( self, *, page_id: str, from_url: str, to_url: str, opener_page_id: str = "", new_page: bool = False, ) -> None: self._pending_navigations[page_id] = { "due": time.monotonic() + self._navigation_reorder_sec, "from": from_url, "to": to_url, "page_id": page_id, "opener_page_id": opener_page_id, "new_page": new_page, } def _flush_pending_navigations(self, *, force: bool = False) -> None: now = time.monotonic() ready = [ page_id for page_id, item in self._pending_navigations.items() if force or float(item.get("due") or 0.0) <= now ] for page_id in ready: self._flush_pending_pointerdowns(force=True, page_id=page_id) item = self._pending_navigations.pop(page_id, None) or {} self.emit( RawUserEvent( kind="navigation", surface="browser", url=str(item.get("to") or ""), meta={ "from": item.get("from") or "", "to": item.get("to") or "", "page_id": page_id, "opener_page_id": item.get("opener_page_id") or "", "new_page": bool(item.get("new_page")), }, ) ) def _install_and_drain(self, page: Any) -> dict[str, Any]: """Runs on Playwright worker thread (once per open tab).""" try: page.evaluate(_LISTEN_SCRIPT) except Exception: pass # Also try frames try: for frame in page.frames: try: frame.evaluate(_LISTEN_SCRIPT) except Exception: pass except Exception: pass events = [] try: events = page.evaluate( """() => { const e = window.__openclawUiInspectorEvents || []; window.__openclawUiInspectorEvents = []; return e; }""" ) or [] except Exception: events = [] url = "" try: url = page.url or "" except Exception: url = "" page_id = "" opener_page_id = "" try: import kit_browser page_id = kit_browser.page_id_for(page) try: opener = page.opener() if opener is not None: opener_page_id = kit_browser.page_id_for(opener) except Exception: pass except Exception: pass return { "url": url, "page_id": page_id, "opener_page_id": opener_page_id, "events": events, } def _emit_dom_event(self, ev: dict[str, Any]) -> None: event_id = str(ev.get("event_id") or "") if event_id: if event_id in self._seen_event_ids: return self._seen_event_ids.add(event_id) self._seen_event_order.append(event_id) if len(self._seen_event_order) > 2000: old = self._seen_event_order.pop(0) self._seen_event_ids.discard(old) if self.on_raw_event: try: self.on_raw_event(dict(ev)) except Exception: pass kind = str(ev.get("kind") or "") hit_target = dict(ev.get("hit_target") or ev.get("target") or {}) candidate_chain = list(ev.get("candidate_chain") or ev.get("ancestry") or []) if not candidate_chain: candidate_chain = [hit_target] action_hint = dict(ev.get("action_target") or {}) if action_hint and action_hint != hit_target: candidate_chain.append(action_hint) try: from core.policies.target_scoring import resolve_browser_target target, target_scores = resolve_browser_target( hit_target, candidate_chain, x=int(ev.get("x")) if ev.get("x") is not None else None, y=int(ev.get("y")) if ev.get("y") is not None else None, ) except Exception: target = dict(ev.get("action_target") or ev.get("target") or hit_target) target_scores = [] if candidate_chain: target["parent_chain"] = candidate_chain frame = { "url": ev.get("frame_url") or "", "name": ev.get("frame_name") or "", "id": ev.get("frame_id") or "", } if kind in {"pointerdown", "click"}: raw = RawUserEvent( kind="click", surface="browser", target=target, frame=frame, url=str(ev.get("url") or ""), x=int(ev.get("x")) if ev.get("x") is not None else None, y=int(ev.get("y")) if ev.get("y") is not None else None, meta={ "button": ev.get("button"), "event_id": event_id, "document_id": ev.get("document_id"), "page_id": ev.get("page_id"), "frame_id": ev.get("frame_id"), "sequence": ev.get("sequence"), "source_ts_ms": ev.get("ts"), "hit_target": ev.get("hit_target") or {}, "action_target": ev.get("action_target") or target, "target_scores": target_scores, "captured_from": kind, }, ) if kind == "pointerdown": key = self._pointer_key(ev) self._pending_pointerdowns[key] = { "due": time.monotonic() + self._pointer_fallback_sec, "event": raw, "page_id": str(ev.get("page_id") or ""), } return self._cancel_pointerdown_for_click(ev) self.emit(raw) elif kind == "input": self.emit( RawUserEvent( kind="input", surface="browser", text=str(ev.get("text") or target.get("value") or ""), target=target, frame=frame, url=str(ev.get("url") or ""), meta={ "input_type": target.get("input_type") or "", "event_id": event_id, "document_id": ev.get("document_id"), "page_id": ev.get("page_id"), "frame_id": ev.get("frame_id"), "sequence": ev.get("sequence"), "source_ts_ms": ev.get("ts"), "input_event": ev.get("input_event") or "input", "is_trusted": bool(ev.get("is_trusted")), }, ) ) elif kind == "key": self.emit( RawUserEvent( kind="key", surface="browser", key=str(ev.get("key") or ""), target=target, frame=frame, url=str(ev.get("url") or ""), meta={ "event_id": event_id, "document_id": ev.get("document_id"), "page_id": ev.get("page_id"), "frame_id": ev.get("frame_id"), "sequence": ev.get("sequence"), "source_ts_ms": ev.get("ts"), }, ) ) @staticmethod def _pointer_key(ev: dict[str, Any]) -> str: return "|".join( str(ev.get(key) or "") for key in ("page_id", "document_id", "pointer_id", "button") ) def _cancel_pointerdown_for_click(self, ev: dict[str, Any]) -> None: exact = self._pointer_key(ev) if exact in self._pending_pointerdowns: self._pending_pointerdowns.pop(exact, None) return page_id = str(ev.get("page_id") or "") button = str(ev.get("button") or "") for key, item in list(self._pending_pointerdowns.items()): event = item.get("event") meta = getattr(event, "meta", {}) or {} if str(meta.get("page_id") or "") == page_id and str(meta.get("button") or "") == button: self._pending_pointerdowns.pop(key, None) return def _flush_pending_pointerdowns( self, *, force: bool = False, page_id: str = "" ) -> None: now = time.monotonic() ready = [ key for key, item in self._pending_pointerdowns.items() if (not page_id or item.get("page_id") == page_id) and (force or float(item.get("due") or 0.0) <= now) ] for key in ready: item = self._pending_pointerdowns.pop(key, None) or {} event = item.get("event") if event is not None: event.meta["pointerdown_fallback"] = True self.emit(event)