Files
skill-template/tools/ui_inspector/recorder/browser.py
chendelian 633ecff4e2 chore: archive tools/ui_inspector snapshot before migration removal.
Keep the last in-template copy on main so history can restore it; product continues in jiangchang-ui-recorder.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-07 11:19:14 +08:00

741 lines
28 KiB
Python

"""Browser recorder for Inspector-owned Playwright sessions.
Injects capture-phase listeners that do NOT preventDefault / stopPropagation
and do NOT mutate business DOM. Re-attaches after navigation.
"""
from __future__ import annotations
import json
import queue
import threading
import time
from typing import Any, Callable, Optional
from .base import RawUserEvent, RecorderBase, RecorderState
_LISTEN_SCRIPT = r"""
(() => {
if (window.__openclawUiInspectorRecorderInstalled) return;
window.__openclawUiInspectorRecorderInstalled = true;
window.__openclawUiInspectorEvents = window.__openclawUiInspectorEvents || [];
window.__openclawUiInspectorDocId = window.__openclawUiInspectorDocId ||
(Date.now().toString(36) + "-" + Math.random().toString(36).slice(2));
window.__openclawUiInspectorSeq = window.__openclawUiInspectorSeq || 0;
function cssPath(el) {
if (!el || !el.tagName) return "";
const esc = (value) => {
try { return CSS.escape(String(value)); } catch (e) { return String(value).replace(/[^a-zA-Z0-9_-]/g, "\\$&"); }
};
const parts = [];
let cur = el;
for (let i = 0; cur && i < 12; i++) {
let part = cur.tagName.toLowerCase();
if (cur.id) { part += "#" + esc(cur.id); parts.unshift(part); break; }
const cls = (cur.className && typeof cur.className === "string")
? cur.className.trim().split(/\s+/).slice(0, 3).map(esc).join(".")
: "";
if (cls) part += "." + cls;
parts.unshift(part);
cur = cur.parentElement;
}
return parts.join(" > ");
}
function stableCss(el) {
if (!el || !el.getAttribute) return "";
const quote = (value) => JSON.stringify(String(value));
const ownId = el.id || "";
if (ownId) return "#" + CSS.escape(ownId);
const testId = el.getAttribute("data-testid") || "";
if (testId) return `[data-testid=${quote(testId)}]`;
const tag = (el.tagName || "").toLowerCase();
let leaf = "";
const href = el.getAttribute("href") || "";
if (tag === "a" && href) leaf = `a[href=${quote(href)}]`;
else if (el.getAttribute("contenteditable") != null) leaf = `${tag || "*"}[contenteditable]`;
else if (el.getAttribute("name")) leaf = `${tag || "*"}[name=${quote(el.getAttribute("name"))}]`;
else if (el.getAttribute("aria-label")) leaf = `${tag || "*"}[aria-label=${quote(el.getAttribute("aria-label"))}]`;
else if (el.getAttribute("placeholder")) leaf = `${tag || "*"}[placeholder=${quote(el.getAttribute("placeholder"))}]`;
const stableClasses = (node) => {
if (!node || typeof node.className !== "string") return [];
return node.className.trim().split(/\s+/).filter((token) => {
if (!token || token.length < 4 || token.length > 64) return false;
const lower = token.toLowerCase();
const semantic = /(button|btn|input|editor|control|action|submit|send|publish|post|play|pause|menu|option|tab|dialog|form|toolbar|container)/.test(lower);
const structured = /[-_]/.test(token) && !/^[a-z0-9_-]{6,14}$/i.test(token);
return semantic || structured;
}).slice(0, 2);
};
if (!leaf) {
const classes = stableClasses(el);
leaf = tag + classes.map(cls => "." + CSS.escape(cls)).join("");
}
let cur = el.parentElement;
const relative = [leaf];
for (let i = 0; cur && i < 10; i++, cur = cur.parentElement) {
if (cur.id) return `#${CSS.escape(cur.id)} > ${relative.reverse().join(" > ")}`;
const ancestorTestId = cur.getAttribute?.("data-testid") || "";
if (ancestorTestId) return `[data-testid=${quote(ancestorTestId)}] > ${relative.reverse().join(" > ")}`;
const curTag = (cur.tagName || "").toLowerCase();
const classes = stableClasses(cur);
relative.push(curTag + classes.map(cls => "." + CSS.escape(cls)).join(""));
}
return "";
}
function editableValue(el) {
if (!el) return "";
if (el.value != null) return String(el.value);
if (el.isContentEditable || el.getAttribute?.("contenteditable") != null) {
return String(el.innerText != null ? el.innerText : (el.textContent || ""));
}
return "";
}
function isEditable(el) {
if (!el || !el.getAttribute) return false;
const tag = (el.tagName || "").toLowerCase();
return tag === "input" || tag === "textarea" || tag === "select" ||
!!el.isContentEditable || el.getAttribute("contenteditable") != null;
}
function editableRoot(el) {
let cur = el;
for (let i = 0; cur && i < 12; i++) {
if (cur.getAttribute && cur.getAttribute("contenteditable") != null) return cur;
const tag = (cur.tagName || "").toLowerCase();
if (tag === "input" || tag === "textarea" || tag === "select") return cur;
cur = cur.parentElement;
}
return null;
}
function semanticRole(el) {
if (!el || !el.getAttribute) return "";
const explicit = el.getAttribute("role");
if (explicit) return explicit;
const tag = (el.tagName || "").toLowerCase();
if (el.isContentEditable) return "textbox";
if (tag === "a") return "link";
if (tag === "textarea") return "textbox";
if (tag === "select") return "combobox";
if (tag === "input") {
const type = (el.type || "text").toLowerCase();
if (type === "checkbox") return "checkbox";
if (type === "radio") return "radio";
if (["button", "submit", "reset"].includes(type)) return "button";
return "textbox";
}
return tag;
}
function actionableFromEvent(ev) {
const path = (ev && typeof ev.composedPath === "function") ? ev.composedPath() : [];
const nodes = path.length ? path : [ev && ev.target];
let best = ev && ev.target;
let bestScore = -1;
for (const el of nodes) {
if (!el || !el.matches) continue;
const tag = (el.tagName || "").toLowerCase();
const role = (el.getAttribute("role") || "").toLowerCase();
const cls = typeof el.className === "string" ? el.className.toLowerCase() : "";
let cursor = "";
try { cursor = getComputedStyle(el).cursor || ""; } catch (e) {}
let score = 0;
if (el.matches("button,a,input,textarea,select,summary,label")) score = 100;
if (el.isContentEditable || el.getAttribute("contenteditable") != null) score = 98;
if (["button","link","textbox","combobox","checkbox","radio","menuitem","option","tab"].includes(role)) score = Math.max(score, 96);
else if (role) score = Math.max(score, 52);
if (el.getAttribute("data-testid") || el.getAttribute("aria-label")) score = Math.max(score, 90);
if (el.getAttribute("tabindex") != null) score = Math.max(score, 74);
if (cursor === "pointer") score = Math.max(score, 68);
// Many production apps use camelCase classes such as sendControl or
// playButton. Prefer that actionable wrapper over its inherited-cursor
// SVG/path child while still retaining hit_target for diagnostics.
if (tag.includes("-") || /(button|btn|submit|send|play|control|action|click)/i.test(cls)) score = Math.max(score, 72);
if (score > bestScore) { best = el; bestScore = score; }
}
return best;
}
function snap(el) {
if (!el || !el.getAttribute) return {};
const role = semanticRole(el);
let cursor = "";
let rect = {};
try {
cursor = getComputedStyle(el).cursor || "";
const r = el.getBoundingClientRect();
rect = {left:r.left, top:r.top, right:r.right, bottom:r.bottom, width:r.width, height:r.height};
} catch (e) {}
const tag = el.tagName ? el.tagName.toLowerCase() : "";
const css = cssPath(el);
const stable = stableCss(el);
let cssCount = null;
let stableCssCount = null;
try { cssCount = css ? document.querySelectorAll(css).length : null; } catch (e) {}
try { stableCssCount = stable ? document.querySelectorAll(stable).length : null; } catch (e) {}
const href = el.getAttribute("href") || "";
let hrefCount = null;
try {
hrefCount = href ? Array.from(document.querySelectorAll("a[href]")).filter(
node => node.getAttribute("href") === href
).length : null;
} catch (e) {}
const roleNames = ["button","link","textbox","combobox","checkbox","radio","menuitem","option","tab"];
return {
role: role,
explicit_role: el.getAttribute("role") || "",
aria_label: el.getAttribute("aria-label") || "",
name: el.getAttribute("aria-label") || el.name || el.id ||
((el.matches?.("button,a,[role],label,summary") || roleNames.includes(role) || cursor === "pointer" || tag.includes("-"))
? (el.innerText || el.textContent || "").trim().slice(0, 120) : ""),
text: (el.innerText || el.textContent || "").trim().slice(0, 120),
value: editableValue(el).slice(0, 500),
test_id: el.getAttribute("data-testid") || "",
name_attr: el.getAttribute("name") || "",
href: href,
href_count: hrefCount,
title: el.getAttribute("title") || "",
alt: el.getAttribute("alt") || "",
id: el.id || "",
placeholder: el.getAttribute("placeholder") || el.getAttribute("aria-placeholder") || "",
label: "",
input_type: (el.type || "").toLowerCase(),
tag: tag,
class_name: (typeof el.className === "string" ? el.className : "").slice(0, 300),
cursor: cursor,
rect: rect,
disabled: !!el.disabled || el.getAttribute("aria-disabled") === "true",
contenteditable: !!el.isContentEditable || el.getAttribute("contenteditable") != null,
contenteditable_attr: el.getAttribute("contenteditable") != null,
tabindex: el.getAttribute("tabindex"),
css: css,
css_count: cssCount,
stable_css: stable,
stable_css_count: stableCssCount,
fingerprint: [
el.tagName?.toLowerCase() || "", el.id || "",
el.getAttribute("data-testid") || "", el.getAttribute("name") || "",
css
].join("|"),
};
}
function ancestry(el) {
const out = [];
let cur = el;
for (let i = 0; cur && i < 12; i++) {
out.push(snap(cur));
cur = cur.parentElement;
}
return out;
}
function push(kind, payload) {
try {
const item = Object.assign({
event_id: window.__openclawUiInspectorDocId + ":" + (++window.__openclawUiInspectorSeq),
document_id: window.__openclawUiInspectorDocId,
sequence: window.__openclawUiInspectorSeq,
kind: kind,
ts: Date.now(),
url: location.href,
title: document.title,
frame_url: location.href,
frame_name: window.name || "",
}, payload || {});
window.__openclawUiInspectorEvents.push(item);
if (typeof window.__openclawUiInspectorEmit === "function") {
try { window.__openclawUiInspectorEmit(item); } catch (e) {}
}
if (window.__openclawUiInspectorEvents.length > 200) {
window.__openclawUiInspectorEvents.splice(0, 50);
}
} catch (e) {}
}
document.addEventListener("click", (ev) => {
const hit = ev.target;
const action = actionableFromEvent(ev) || hit;
push("click", {
target: snap(action),
hit_target: snap(hit),
action_target: snap(action),
ancestry: ancestry(hit),
candidate_chain: ancestry(hit),
button: ev.button,
x: ev.clientX,
y: ev.clientY
});
}, true);
document.addEventListener("pointerdown", (ev) => {
const hit = ev.target;
push("pointerdown", {
target: snap(hit),
hit_target: snap(hit),
candidate_chain: ancestry(hit),
button: ev.button,
pointer_id: ev.pointerId,
pointer_type: ev.pointerType || "",
x: ev.clientX,
y: ev.clientY
});
}, true);
document.addEventListener("input", (ev) => {
if (ev.isComposing) return;
const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target;
push("input", {
target: snap(t), text: editableValue(t), input_event: ev.inputType || "input",
candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted
});
}, true);
document.addEventListener("change", (ev) => {
const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target;
push("input", { target: snap(t), text: editableValue(t), input_event: "change",
candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted });
}, true);
document.addEventListener("compositionend", (ev) => {
const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target;
push("input", {
target: snap(t), text: editableValue(t), input_event: "compositionend",
candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted
});
}, true);
document.addEventListener("paste", (ev) => {
const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target;
setTimeout(() => push("input", {
target: snap(t), text: editableValue(t), input_event: "paste",
candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted
}), 0);
}, true);
document.addEventListener("keydown", (ev) => {
const special = ["Enter","Escape","Tab","Backspace","Delete"];
const editing = isEditable(ev.target);
const mutatingEditKey = editing && (ev.key === "Backspace" || ev.key === "Delete");
if (!mutatingEditKey && (special.indexOf(ev.key) >= 0 || (ev.key && ev.key.length > 1))) {
const t = actionableFromEvent(ev) || ev.target;
push("key", { key: ev.key, target: snap(t) });
}
}, true);
})();
"""
class BrowserRecorder(RecorderBase):
capability_map = {
"click": "implemented",
"fill": "implemented",
"press_key": "implemented",
"navigation": "partial",
}
def __init__(
self,
*,
on_event: Optional[Callable[[RawUserEvent], None]] = None,
on_raw_event: Optional[Callable[[dict[str, Any]], None]] = None,
poll_sec: float = 0.25,
) -> None:
super().__init__(on_event=on_event)
self.on_raw_event = on_raw_event
self.poll_sec = poll_sec
self._last_url = ""
self._last_urls_by_page: dict[str, str] = {}
self._installed_pages: set[int] = set()
self._bound_events: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=512)
self._seen_event_ids: set[str] = set()
self._seen_event_order: list[str] = []
self._known_page_ids: set[str] = set()
self._pending_navigations: dict[str, dict[str, Any]] = {}
self._pending_pointerdowns: dict[str, dict[str, Any]] = {}
self._navigation_reorder_sec = 0.45
self._pointer_fallback_sec = 0.35
@property
def capability(self) -> dict[str, str]:
return dict(self.capability_map)
def start(self) -> None:
if self._thread and self._thread.is_alive():
self.status = RecorderState.RECORDING
return
self._stop.clear()
self.status = RecorderState.RECORDING
try:
import kit_browser
kit_browser.install_event_bridge(_LISTEN_SCRIPT, self._receive_bound_event)
initial_ids = kit_browser.call_on_each_open_page(
lambda page: kit_browser.page_id_for(page)
)
self._known_page_ids.update(
str(page_id) for page_id in initial_ids if page_id
)
except Exception:
# Polling remains as a compatibility fallback.
pass
self._thread = threading.Thread(
target=self._loop, name="ui-inspector-browser-recorder", daemon=True
)
self._thread.start()
def stop(self) -> None:
self._stop.set()
if self._thread and self._thread.is_alive():
self._thread.join(timeout=2.0)
self._thread = None
# Disable the bridge before draining its local queue so an event that
# arrived just before Stop is not silently lost, while no new events
# can race with the drain.
try:
import kit_browser
kit_browser.clear_event_bridge()
except Exception:
pass
if self.status == RecorderState.RECORDING:
try:
self._drain_bound_events()
self._flush_pending_pointerdowns(force=True)
self._flush_pending_navigations(force=True)
except Exception:
pass
self.status = RecorderState.STOPPED
def _receive_bound_event(self, event: dict[str, Any]) -> None:
try:
self._bound_events.put_nowait(dict(event or {}))
except queue.Full:
try:
self._bound_events.get_nowait()
self._bound_events.put_nowait(dict(event or {}))
except (queue.Empty, queue.Full):
pass
def _loop(self) -> None:
while not self._stop.is_set():
if self.status == RecorderState.PAUSED:
time.sleep(0.1)
continue
if self.status != RecorderState.RECORDING:
time.sleep(0.05)
continue
try:
self._tick()
except Exception:
pass
time.sleep(self.poll_sec)
def _tick(self) -> None:
import kit_browser
self._drain_bound_events()
# Install + drain on every open tab (detail may be a second tab).
payloads = kit_browser.call_on_each_open_page(self._install_and_drain)
if payloads:
for payload in payloads:
if not isinstance(payload, dict):
continue
page_id = str(payload.get("page_id") or "")
url = str(payload.get("url") or "")
previous_url = self._last_urls_by_page.get(page_id, "") if page_id else ""
if previous_url and url and url != previous_url:
self._schedule_navigation(
page_id=page_id, from_url=previous_url, to_url=url
)
elif (
page_id
and page_id not in self._known_page_ids
and self._known_page_ids
and url
and not url.startswith("about:")
):
opener_id = str(payload.get("opener_page_id") or "")
self._schedule_navigation(
page_id=page_id,
from_url=self._last_urls_by_page.get(opener_id, ""),
to_url=url,
opener_page_id=opener_id,
new_page=True,
)
if page_id and url:
self._last_urls_by_page[page_id] = url
self._known_page_ids.add(page_id)
for ev in payload.get("events") or []:
if isinstance(ev, dict) and not ev.get("page_id"):
ev["page_id"] = payload.get("page_id") or ""
self._emit_dom_event(ev)
self._flush_pending_pointerdowns()
# Navigation is intentionally emitted after both bridge and polling
# queues have been drained. This preserves click -> navigation order
# even when the context bridge is unavailable and polling is used.
self._flush_pending_navigations()
def _drain_bound_events(self) -> None:
while True:
try:
self._emit_dom_event(self._bound_events.get_nowait())
except queue.Empty:
break
def _schedule_navigation(
self,
*,
page_id: str,
from_url: str,
to_url: str,
opener_page_id: str = "",
new_page: bool = False,
) -> None:
self._pending_navigations[page_id] = {
"due": time.monotonic() + self._navigation_reorder_sec,
"from": from_url,
"to": to_url,
"page_id": page_id,
"opener_page_id": opener_page_id,
"new_page": new_page,
}
def _flush_pending_navigations(self, *, force: bool = False) -> None:
now = time.monotonic()
ready = [
page_id
for page_id, item in self._pending_navigations.items()
if force or float(item.get("due") or 0.0) <= now
]
for page_id in ready:
self._flush_pending_pointerdowns(force=True, page_id=page_id)
item = self._pending_navigations.pop(page_id, None) or {}
self.emit(
RawUserEvent(
kind="navigation",
surface="browser",
url=str(item.get("to") or ""),
meta={
"from": item.get("from") or "",
"to": item.get("to") or "",
"page_id": page_id,
"opener_page_id": item.get("opener_page_id") or "",
"new_page": bool(item.get("new_page")),
},
)
)
def _install_and_drain(self, page: Any) -> dict[str, Any]:
"""Runs on Playwright worker thread (once per open tab)."""
try:
page.evaluate(_LISTEN_SCRIPT)
except Exception:
pass
# Also try frames
try:
for frame in page.frames:
try:
frame.evaluate(_LISTEN_SCRIPT)
except Exception:
pass
except Exception:
pass
events = []
try:
events = page.evaluate(
"""() => {
const e = window.__openclawUiInspectorEvents || [];
window.__openclawUiInspectorEvents = [];
return e;
}"""
) or []
except Exception:
events = []
url = ""
try:
url = page.url or ""
except Exception:
url = ""
page_id = ""
opener_page_id = ""
try:
import kit_browser
page_id = kit_browser.page_id_for(page)
try:
opener = page.opener()
if opener is not None:
opener_page_id = kit_browser.page_id_for(opener)
except Exception:
pass
except Exception:
pass
return {
"url": url,
"page_id": page_id,
"opener_page_id": opener_page_id,
"events": events,
}
def _emit_dom_event(self, ev: dict[str, Any]) -> None:
event_id = str(ev.get("event_id") or "")
if event_id:
if event_id in self._seen_event_ids:
return
self._seen_event_ids.add(event_id)
self._seen_event_order.append(event_id)
if len(self._seen_event_order) > 2000:
old = self._seen_event_order.pop(0)
self._seen_event_ids.discard(old)
if self.on_raw_event:
try:
self.on_raw_event(dict(ev))
except Exception:
pass
kind = str(ev.get("kind") or "")
hit_target = dict(ev.get("hit_target") or ev.get("target") or {})
candidate_chain = list(ev.get("candidate_chain") or ev.get("ancestry") or [])
if not candidate_chain:
candidate_chain = [hit_target]
action_hint = dict(ev.get("action_target") or {})
if action_hint and action_hint != hit_target:
candidate_chain.append(action_hint)
try:
from core.policies.target_scoring import resolve_browser_target
target, target_scores = resolve_browser_target(
hit_target,
candidate_chain,
x=int(ev.get("x")) if ev.get("x") is not None else None,
y=int(ev.get("y")) if ev.get("y") is not None else None,
)
except Exception:
target = dict(ev.get("action_target") or ev.get("target") or hit_target)
target_scores = []
if candidate_chain:
target["parent_chain"] = candidate_chain
frame = {
"url": ev.get("frame_url") or "",
"name": ev.get("frame_name") or "",
"id": ev.get("frame_id") or "",
}
if kind in {"pointerdown", "click"}:
raw = RawUserEvent(
kind="click",
surface="browser",
target=target,
frame=frame,
url=str(ev.get("url") or ""),
x=int(ev.get("x")) if ev.get("x") is not None else None,
y=int(ev.get("y")) if ev.get("y") is not None else None,
meta={
"button": ev.get("button"),
"event_id": event_id,
"document_id": ev.get("document_id"),
"page_id": ev.get("page_id"),
"frame_id": ev.get("frame_id"),
"sequence": ev.get("sequence"),
"source_ts_ms": ev.get("ts"),
"hit_target": ev.get("hit_target") or {},
"action_target": ev.get("action_target") or target,
"target_scores": target_scores,
"captured_from": kind,
},
)
if kind == "pointerdown":
key = self._pointer_key(ev)
self._pending_pointerdowns[key] = {
"due": time.monotonic() + self._pointer_fallback_sec,
"event": raw,
"page_id": str(ev.get("page_id") or ""),
}
return
self._cancel_pointerdown_for_click(ev)
self.emit(raw)
elif kind == "input":
self.emit(
RawUserEvent(
kind="input",
surface="browser",
text=str(ev.get("text") or target.get("value") or ""),
target=target,
frame=frame,
url=str(ev.get("url") or ""),
meta={
"input_type": target.get("input_type") or "",
"event_id": event_id,
"document_id": ev.get("document_id"),
"page_id": ev.get("page_id"),
"frame_id": ev.get("frame_id"),
"sequence": ev.get("sequence"),
"source_ts_ms": ev.get("ts"),
"input_event": ev.get("input_event") or "input",
"is_trusted": bool(ev.get("is_trusted")),
},
)
)
elif kind == "key":
self.emit(
RawUserEvent(
kind="key",
surface="browser",
key=str(ev.get("key") or ""),
target=target,
frame=frame,
url=str(ev.get("url") or ""),
meta={
"event_id": event_id,
"document_id": ev.get("document_id"),
"page_id": ev.get("page_id"),
"frame_id": ev.get("frame_id"),
"sequence": ev.get("sequence"),
"source_ts_ms": ev.get("ts"),
},
)
)
@staticmethod
def _pointer_key(ev: dict[str, Any]) -> str:
return "|".join(
str(ev.get(key) or "")
for key in ("page_id", "document_id", "pointer_id", "button")
)
def _cancel_pointerdown_for_click(self, ev: dict[str, Any]) -> None:
exact = self._pointer_key(ev)
if exact in self._pending_pointerdowns:
self._pending_pointerdowns.pop(exact, None)
return
page_id = str(ev.get("page_id") or "")
button = str(ev.get("button") or "")
for key, item in list(self._pending_pointerdowns.items()):
event = item.get("event")
meta = getattr(event, "meta", {}) or {}
if str(meta.get("page_id") or "") == page_id and str(meta.get("button") or "") == button:
self._pending_pointerdowns.pop(key, None)
return
def _flush_pending_pointerdowns(
self, *, force: bool = False, page_id: str = ""
) -> None:
now = time.monotonic()
ready = [
key for key, item in self._pending_pointerdowns.items()
if (not page_id or item.get("page_id") == page_id)
and (force or float(item.get("due") or 0.0) <= now)
]
for key in ready:
item = self._pending_pointerdowns.pop(key, None) or {}
event = item.get("event")
if event is not None:
event.meta["pointerdown_fallback"] = True
self.emit(event)