Keep the last in-template copy on main so history can restore it; product continues in jiangchang-ui-recorder. Co-authored-by: Cursor <cursoragent@cursor.com>
741 lines
28 KiB
Python
741 lines
28 KiB
Python
"""Browser recorder for Inspector-owned Playwright sessions.
|
|
|
|
Injects capture-phase listeners that do NOT preventDefault / stopPropagation
|
|
and do NOT mutate business DOM. Re-attaches after navigation.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import queue
|
|
import threading
|
|
import time
|
|
from typing import Any, Callable, Optional
|
|
|
|
from .base import RawUserEvent, RecorderBase, RecorderState
|
|
|
|
_LISTEN_SCRIPT = r"""
|
|
(() => {
|
|
if (window.__openclawUiInspectorRecorderInstalled) return;
|
|
window.__openclawUiInspectorRecorderInstalled = true;
|
|
window.__openclawUiInspectorEvents = window.__openclawUiInspectorEvents || [];
|
|
window.__openclawUiInspectorDocId = window.__openclawUiInspectorDocId ||
|
|
(Date.now().toString(36) + "-" + Math.random().toString(36).slice(2));
|
|
window.__openclawUiInspectorSeq = window.__openclawUiInspectorSeq || 0;
|
|
|
|
function cssPath(el) {
|
|
if (!el || !el.tagName) return "";
|
|
const esc = (value) => {
|
|
try { return CSS.escape(String(value)); } catch (e) { return String(value).replace(/[^a-zA-Z0-9_-]/g, "\\$&"); }
|
|
};
|
|
const parts = [];
|
|
let cur = el;
|
|
for (let i = 0; cur && i < 12; i++) {
|
|
let part = cur.tagName.toLowerCase();
|
|
if (cur.id) { part += "#" + esc(cur.id); parts.unshift(part); break; }
|
|
const cls = (cur.className && typeof cur.className === "string")
|
|
? cur.className.trim().split(/\s+/).slice(0, 3).map(esc).join(".")
|
|
: "";
|
|
if (cls) part += "." + cls;
|
|
parts.unshift(part);
|
|
cur = cur.parentElement;
|
|
}
|
|
return parts.join(" > ");
|
|
}
|
|
|
|
function stableCss(el) {
|
|
if (!el || !el.getAttribute) return "";
|
|
const quote = (value) => JSON.stringify(String(value));
|
|
const ownId = el.id || "";
|
|
if (ownId) return "#" + CSS.escape(ownId);
|
|
const testId = el.getAttribute("data-testid") || "";
|
|
if (testId) return `[data-testid=${quote(testId)}]`;
|
|
const tag = (el.tagName || "").toLowerCase();
|
|
let leaf = "";
|
|
const href = el.getAttribute("href") || "";
|
|
if (tag === "a" && href) leaf = `a[href=${quote(href)}]`;
|
|
else if (el.getAttribute("contenteditable") != null) leaf = `${tag || "*"}[contenteditable]`;
|
|
else if (el.getAttribute("name")) leaf = `${tag || "*"}[name=${quote(el.getAttribute("name"))}]`;
|
|
else if (el.getAttribute("aria-label")) leaf = `${tag || "*"}[aria-label=${quote(el.getAttribute("aria-label"))}]`;
|
|
else if (el.getAttribute("placeholder")) leaf = `${tag || "*"}[placeholder=${quote(el.getAttribute("placeholder"))}]`;
|
|
const stableClasses = (node) => {
|
|
if (!node || typeof node.className !== "string") return [];
|
|
return node.className.trim().split(/\s+/).filter((token) => {
|
|
if (!token || token.length < 4 || token.length > 64) return false;
|
|
const lower = token.toLowerCase();
|
|
const semantic = /(button|btn|input|editor|control|action|submit|send|publish|post|play|pause|menu|option|tab|dialog|form|toolbar|container)/.test(lower);
|
|
const structured = /[-_]/.test(token) && !/^[a-z0-9_-]{6,14}$/i.test(token);
|
|
return semantic || structured;
|
|
}).slice(0, 2);
|
|
};
|
|
if (!leaf) {
|
|
const classes = stableClasses(el);
|
|
leaf = tag + classes.map(cls => "." + CSS.escape(cls)).join("");
|
|
}
|
|
let cur = el.parentElement;
|
|
const relative = [leaf];
|
|
for (let i = 0; cur && i < 10; i++, cur = cur.parentElement) {
|
|
if (cur.id) return `#${CSS.escape(cur.id)} > ${relative.reverse().join(" > ")}`;
|
|
const ancestorTestId = cur.getAttribute?.("data-testid") || "";
|
|
if (ancestorTestId) return `[data-testid=${quote(ancestorTestId)}] > ${relative.reverse().join(" > ")}`;
|
|
const curTag = (cur.tagName || "").toLowerCase();
|
|
const classes = stableClasses(cur);
|
|
relative.push(curTag + classes.map(cls => "." + CSS.escape(cls)).join(""));
|
|
}
|
|
return "";
|
|
}
|
|
|
|
function editableValue(el) {
|
|
if (!el) return "";
|
|
if (el.value != null) return String(el.value);
|
|
if (el.isContentEditable || el.getAttribute?.("contenteditable") != null) {
|
|
return String(el.innerText != null ? el.innerText : (el.textContent || ""));
|
|
}
|
|
return "";
|
|
}
|
|
|
|
function isEditable(el) {
|
|
if (!el || !el.getAttribute) return false;
|
|
const tag = (el.tagName || "").toLowerCase();
|
|
return tag === "input" || tag === "textarea" || tag === "select" ||
|
|
!!el.isContentEditable || el.getAttribute("contenteditable") != null;
|
|
}
|
|
|
|
function editableRoot(el) {
|
|
let cur = el;
|
|
for (let i = 0; cur && i < 12; i++) {
|
|
if (cur.getAttribute && cur.getAttribute("contenteditable") != null) return cur;
|
|
const tag = (cur.tagName || "").toLowerCase();
|
|
if (tag === "input" || tag === "textarea" || tag === "select") return cur;
|
|
cur = cur.parentElement;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function semanticRole(el) {
|
|
if (!el || !el.getAttribute) return "";
|
|
const explicit = el.getAttribute("role");
|
|
if (explicit) return explicit;
|
|
const tag = (el.tagName || "").toLowerCase();
|
|
if (el.isContentEditable) return "textbox";
|
|
if (tag === "a") return "link";
|
|
if (tag === "textarea") return "textbox";
|
|
if (tag === "select") return "combobox";
|
|
if (tag === "input") {
|
|
const type = (el.type || "text").toLowerCase();
|
|
if (type === "checkbox") return "checkbox";
|
|
if (type === "radio") return "radio";
|
|
if (["button", "submit", "reset"].includes(type)) return "button";
|
|
return "textbox";
|
|
}
|
|
return tag;
|
|
}
|
|
|
|
function actionableFromEvent(ev) {
|
|
const path = (ev && typeof ev.composedPath === "function") ? ev.composedPath() : [];
|
|
const nodes = path.length ? path : [ev && ev.target];
|
|
let best = ev && ev.target;
|
|
let bestScore = -1;
|
|
for (const el of nodes) {
|
|
if (!el || !el.matches) continue;
|
|
const tag = (el.tagName || "").toLowerCase();
|
|
const role = (el.getAttribute("role") || "").toLowerCase();
|
|
const cls = typeof el.className === "string" ? el.className.toLowerCase() : "";
|
|
let cursor = "";
|
|
try { cursor = getComputedStyle(el).cursor || ""; } catch (e) {}
|
|
let score = 0;
|
|
if (el.matches("button,a,input,textarea,select,summary,label")) score = 100;
|
|
if (el.isContentEditable || el.getAttribute("contenteditable") != null) score = 98;
|
|
if (["button","link","textbox","combobox","checkbox","radio","menuitem","option","tab"].includes(role)) score = Math.max(score, 96);
|
|
else if (role) score = Math.max(score, 52);
|
|
if (el.getAttribute("data-testid") || el.getAttribute("aria-label")) score = Math.max(score, 90);
|
|
if (el.getAttribute("tabindex") != null) score = Math.max(score, 74);
|
|
if (cursor === "pointer") score = Math.max(score, 68);
|
|
// Many production apps use camelCase classes such as sendControl or
|
|
// playButton. Prefer that actionable wrapper over its inherited-cursor
|
|
// SVG/path child while still retaining hit_target for diagnostics.
|
|
if (tag.includes("-") || /(button|btn|submit|send|play|control|action|click)/i.test(cls)) score = Math.max(score, 72);
|
|
if (score > bestScore) { best = el; bestScore = score; }
|
|
}
|
|
return best;
|
|
}
|
|
|
|
function snap(el) {
|
|
if (!el || !el.getAttribute) return {};
|
|
const role = semanticRole(el);
|
|
let cursor = "";
|
|
let rect = {};
|
|
try {
|
|
cursor = getComputedStyle(el).cursor || "";
|
|
const r = el.getBoundingClientRect();
|
|
rect = {left:r.left, top:r.top, right:r.right, bottom:r.bottom, width:r.width, height:r.height};
|
|
} catch (e) {}
|
|
const tag = el.tagName ? el.tagName.toLowerCase() : "";
|
|
const css = cssPath(el);
|
|
const stable = stableCss(el);
|
|
let cssCount = null;
|
|
let stableCssCount = null;
|
|
try { cssCount = css ? document.querySelectorAll(css).length : null; } catch (e) {}
|
|
try { stableCssCount = stable ? document.querySelectorAll(stable).length : null; } catch (e) {}
|
|
const href = el.getAttribute("href") || "";
|
|
let hrefCount = null;
|
|
try {
|
|
hrefCount = href ? Array.from(document.querySelectorAll("a[href]")).filter(
|
|
node => node.getAttribute("href") === href
|
|
).length : null;
|
|
} catch (e) {}
|
|
const roleNames = ["button","link","textbox","combobox","checkbox","radio","menuitem","option","tab"];
|
|
return {
|
|
role: role,
|
|
explicit_role: el.getAttribute("role") || "",
|
|
aria_label: el.getAttribute("aria-label") || "",
|
|
name: el.getAttribute("aria-label") || el.name || el.id ||
|
|
((el.matches?.("button,a,[role],label,summary") || roleNames.includes(role) || cursor === "pointer" || tag.includes("-"))
|
|
? (el.innerText || el.textContent || "").trim().slice(0, 120) : ""),
|
|
text: (el.innerText || el.textContent || "").trim().slice(0, 120),
|
|
value: editableValue(el).slice(0, 500),
|
|
test_id: el.getAttribute("data-testid") || "",
|
|
name_attr: el.getAttribute("name") || "",
|
|
href: href,
|
|
href_count: hrefCount,
|
|
title: el.getAttribute("title") || "",
|
|
alt: el.getAttribute("alt") || "",
|
|
id: el.id || "",
|
|
placeholder: el.getAttribute("placeholder") || el.getAttribute("aria-placeholder") || "",
|
|
label: "",
|
|
input_type: (el.type || "").toLowerCase(),
|
|
tag: tag,
|
|
class_name: (typeof el.className === "string" ? el.className : "").slice(0, 300),
|
|
cursor: cursor,
|
|
rect: rect,
|
|
disabled: !!el.disabled || el.getAttribute("aria-disabled") === "true",
|
|
contenteditable: !!el.isContentEditable || el.getAttribute("contenteditable") != null,
|
|
contenteditable_attr: el.getAttribute("contenteditable") != null,
|
|
tabindex: el.getAttribute("tabindex"),
|
|
css: css,
|
|
css_count: cssCount,
|
|
stable_css: stable,
|
|
stable_css_count: stableCssCount,
|
|
fingerprint: [
|
|
el.tagName?.toLowerCase() || "", el.id || "",
|
|
el.getAttribute("data-testid") || "", el.getAttribute("name") || "",
|
|
css
|
|
].join("|"),
|
|
};
|
|
}
|
|
|
|
function ancestry(el) {
|
|
const out = [];
|
|
let cur = el;
|
|
for (let i = 0; cur && i < 12; i++) {
|
|
out.push(snap(cur));
|
|
cur = cur.parentElement;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function push(kind, payload) {
|
|
try {
|
|
const item = Object.assign({
|
|
event_id: window.__openclawUiInspectorDocId + ":" + (++window.__openclawUiInspectorSeq),
|
|
document_id: window.__openclawUiInspectorDocId,
|
|
sequence: window.__openclawUiInspectorSeq,
|
|
kind: kind,
|
|
ts: Date.now(),
|
|
url: location.href,
|
|
title: document.title,
|
|
frame_url: location.href,
|
|
frame_name: window.name || "",
|
|
}, payload || {});
|
|
window.__openclawUiInspectorEvents.push(item);
|
|
if (typeof window.__openclawUiInspectorEmit === "function") {
|
|
try { window.__openclawUiInspectorEmit(item); } catch (e) {}
|
|
}
|
|
if (window.__openclawUiInspectorEvents.length > 200) {
|
|
window.__openclawUiInspectorEvents.splice(0, 50);
|
|
}
|
|
} catch (e) {}
|
|
}
|
|
|
|
document.addEventListener("click", (ev) => {
|
|
const hit = ev.target;
|
|
const action = actionableFromEvent(ev) || hit;
|
|
push("click", {
|
|
target: snap(action),
|
|
hit_target: snap(hit),
|
|
action_target: snap(action),
|
|
ancestry: ancestry(hit),
|
|
candidate_chain: ancestry(hit),
|
|
button: ev.button,
|
|
x: ev.clientX,
|
|
y: ev.clientY
|
|
});
|
|
}, true);
|
|
|
|
document.addEventListener("pointerdown", (ev) => {
|
|
const hit = ev.target;
|
|
push("pointerdown", {
|
|
target: snap(hit),
|
|
hit_target: snap(hit),
|
|
candidate_chain: ancestry(hit),
|
|
button: ev.button,
|
|
pointer_id: ev.pointerId,
|
|
pointer_type: ev.pointerType || "",
|
|
x: ev.clientX,
|
|
y: ev.clientY
|
|
});
|
|
}, true);
|
|
|
|
document.addEventListener("input", (ev) => {
|
|
if (ev.isComposing) return;
|
|
const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target;
|
|
push("input", {
|
|
target: snap(t), text: editableValue(t), input_event: ev.inputType || "input",
|
|
candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted
|
|
});
|
|
}, true);
|
|
|
|
document.addEventListener("change", (ev) => {
|
|
const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target;
|
|
push("input", { target: snap(t), text: editableValue(t), input_event: "change",
|
|
candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted });
|
|
}, true);
|
|
|
|
document.addEventListener("compositionend", (ev) => {
|
|
const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target;
|
|
push("input", {
|
|
target: snap(t), text: editableValue(t), input_event: "compositionend",
|
|
candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted
|
|
});
|
|
}, true);
|
|
|
|
document.addEventListener("paste", (ev) => {
|
|
const t = editableRoot(ev.target) || actionableFromEvent(ev) || ev.target;
|
|
setTimeout(() => push("input", {
|
|
target: snap(t), text: editableValue(t), input_event: "paste",
|
|
candidate_chain: ancestry(ev.target), is_trusted: !!ev.isTrusted
|
|
}), 0);
|
|
}, true);
|
|
|
|
document.addEventListener("keydown", (ev) => {
|
|
const special = ["Enter","Escape","Tab","Backspace","Delete"];
|
|
const editing = isEditable(ev.target);
|
|
const mutatingEditKey = editing && (ev.key === "Backspace" || ev.key === "Delete");
|
|
if (!mutatingEditKey && (special.indexOf(ev.key) >= 0 || (ev.key && ev.key.length > 1))) {
|
|
const t = actionableFromEvent(ev) || ev.target;
|
|
push("key", { key: ev.key, target: snap(t) });
|
|
}
|
|
}, true);
|
|
})();
|
|
"""
|
|
|
|
|
|
class BrowserRecorder(RecorderBase):
|
|
capability_map = {
|
|
"click": "implemented",
|
|
"fill": "implemented",
|
|
"press_key": "implemented",
|
|
"navigation": "partial",
|
|
}
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
on_event: Optional[Callable[[RawUserEvent], None]] = None,
|
|
on_raw_event: Optional[Callable[[dict[str, Any]], None]] = None,
|
|
poll_sec: float = 0.25,
|
|
) -> None:
|
|
super().__init__(on_event=on_event)
|
|
self.on_raw_event = on_raw_event
|
|
self.poll_sec = poll_sec
|
|
self._last_url = ""
|
|
self._last_urls_by_page: dict[str, str] = {}
|
|
self._installed_pages: set[int] = set()
|
|
self._bound_events: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=512)
|
|
self._seen_event_ids: set[str] = set()
|
|
self._seen_event_order: list[str] = []
|
|
self._known_page_ids: set[str] = set()
|
|
self._pending_navigations: dict[str, dict[str, Any]] = {}
|
|
self._pending_pointerdowns: dict[str, dict[str, Any]] = {}
|
|
self._navigation_reorder_sec = 0.45
|
|
self._pointer_fallback_sec = 0.35
|
|
|
|
@property
|
|
def capability(self) -> dict[str, str]:
|
|
return dict(self.capability_map)
|
|
|
|
def start(self) -> None:
|
|
if self._thread and self._thread.is_alive():
|
|
self.status = RecorderState.RECORDING
|
|
return
|
|
self._stop.clear()
|
|
self.status = RecorderState.RECORDING
|
|
try:
|
|
import kit_browser
|
|
|
|
kit_browser.install_event_bridge(_LISTEN_SCRIPT, self._receive_bound_event)
|
|
initial_ids = kit_browser.call_on_each_open_page(
|
|
lambda page: kit_browser.page_id_for(page)
|
|
)
|
|
self._known_page_ids.update(
|
|
str(page_id) for page_id in initial_ids if page_id
|
|
)
|
|
except Exception:
|
|
# Polling remains as a compatibility fallback.
|
|
pass
|
|
self._thread = threading.Thread(
|
|
target=self._loop, name="ui-inspector-browser-recorder", daemon=True
|
|
)
|
|
self._thread.start()
|
|
|
|
def stop(self) -> None:
|
|
self._stop.set()
|
|
if self._thread and self._thread.is_alive():
|
|
self._thread.join(timeout=2.0)
|
|
self._thread = None
|
|
# Disable the bridge before draining its local queue so an event that
|
|
# arrived just before Stop is not silently lost, while no new events
|
|
# can race with the drain.
|
|
try:
|
|
import kit_browser
|
|
|
|
kit_browser.clear_event_bridge()
|
|
except Exception:
|
|
pass
|
|
if self.status == RecorderState.RECORDING:
|
|
try:
|
|
self._drain_bound_events()
|
|
self._flush_pending_pointerdowns(force=True)
|
|
self._flush_pending_navigations(force=True)
|
|
except Exception:
|
|
pass
|
|
self.status = RecorderState.STOPPED
|
|
|
|
def _receive_bound_event(self, event: dict[str, Any]) -> None:
|
|
try:
|
|
self._bound_events.put_nowait(dict(event or {}))
|
|
except queue.Full:
|
|
try:
|
|
self._bound_events.get_nowait()
|
|
self._bound_events.put_nowait(dict(event or {}))
|
|
except (queue.Empty, queue.Full):
|
|
pass
|
|
|
|
def _loop(self) -> None:
|
|
while not self._stop.is_set():
|
|
if self.status == RecorderState.PAUSED:
|
|
time.sleep(0.1)
|
|
continue
|
|
if self.status != RecorderState.RECORDING:
|
|
time.sleep(0.05)
|
|
continue
|
|
try:
|
|
self._tick()
|
|
except Exception:
|
|
pass
|
|
time.sleep(self.poll_sec)
|
|
|
|
def _tick(self) -> None:
|
|
import kit_browser
|
|
|
|
self._drain_bound_events()
|
|
|
|
# Install + drain on every open tab (detail may be a second tab).
|
|
payloads = kit_browser.call_on_each_open_page(self._install_and_drain)
|
|
if payloads:
|
|
for payload in payloads:
|
|
if not isinstance(payload, dict):
|
|
continue
|
|
page_id = str(payload.get("page_id") or "")
|
|
url = str(payload.get("url") or "")
|
|
previous_url = self._last_urls_by_page.get(page_id, "") if page_id else ""
|
|
if previous_url and url and url != previous_url:
|
|
self._schedule_navigation(
|
|
page_id=page_id, from_url=previous_url, to_url=url
|
|
)
|
|
elif (
|
|
page_id
|
|
and page_id not in self._known_page_ids
|
|
and self._known_page_ids
|
|
and url
|
|
and not url.startswith("about:")
|
|
):
|
|
opener_id = str(payload.get("opener_page_id") or "")
|
|
self._schedule_navigation(
|
|
page_id=page_id,
|
|
from_url=self._last_urls_by_page.get(opener_id, ""),
|
|
to_url=url,
|
|
opener_page_id=opener_id,
|
|
new_page=True,
|
|
)
|
|
if page_id and url:
|
|
self._last_urls_by_page[page_id] = url
|
|
self._known_page_ids.add(page_id)
|
|
for ev in payload.get("events") or []:
|
|
if isinstance(ev, dict) and not ev.get("page_id"):
|
|
ev["page_id"] = payload.get("page_id") or ""
|
|
self._emit_dom_event(ev)
|
|
|
|
self._flush_pending_pointerdowns()
|
|
|
|
# Navigation is intentionally emitted after both bridge and polling
|
|
# queues have been drained. This preserves click -> navigation order
|
|
# even when the context bridge is unavailable and polling is used.
|
|
self._flush_pending_navigations()
|
|
|
|
def _drain_bound_events(self) -> None:
|
|
while True:
|
|
try:
|
|
self._emit_dom_event(self._bound_events.get_nowait())
|
|
except queue.Empty:
|
|
break
|
|
|
|
def _schedule_navigation(
|
|
self,
|
|
*,
|
|
page_id: str,
|
|
from_url: str,
|
|
to_url: str,
|
|
opener_page_id: str = "",
|
|
new_page: bool = False,
|
|
) -> None:
|
|
self._pending_navigations[page_id] = {
|
|
"due": time.monotonic() + self._navigation_reorder_sec,
|
|
"from": from_url,
|
|
"to": to_url,
|
|
"page_id": page_id,
|
|
"opener_page_id": opener_page_id,
|
|
"new_page": new_page,
|
|
}
|
|
|
|
def _flush_pending_navigations(self, *, force: bool = False) -> None:
|
|
now = time.monotonic()
|
|
ready = [
|
|
page_id
|
|
for page_id, item in self._pending_navigations.items()
|
|
if force or float(item.get("due") or 0.0) <= now
|
|
]
|
|
for page_id in ready:
|
|
self._flush_pending_pointerdowns(force=True, page_id=page_id)
|
|
item = self._pending_navigations.pop(page_id, None) or {}
|
|
self.emit(
|
|
RawUserEvent(
|
|
kind="navigation",
|
|
surface="browser",
|
|
url=str(item.get("to") or ""),
|
|
meta={
|
|
"from": item.get("from") or "",
|
|
"to": item.get("to") or "",
|
|
"page_id": page_id,
|
|
"opener_page_id": item.get("opener_page_id") or "",
|
|
"new_page": bool(item.get("new_page")),
|
|
},
|
|
)
|
|
)
|
|
|
|
def _install_and_drain(self, page: Any) -> dict[str, Any]:
|
|
"""Runs on Playwright worker thread (once per open tab)."""
|
|
try:
|
|
page.evaluate(_LISTEN_SCRIPT)
|
|
except Exception:
|
|
pass
|
|
# Also try frames
|
|
try:
|
|
for frame in page.frames:
|
|
try:
|
|
frame.evaluate(_LISTEN_SCRIPT)
|
|
except Exception:
|
|
pass
|
|
except Exception:
|
|
pass
|
|
events = []
|
|
try:
|
|
events = page.evaluate(
|
|
"""() => {
|
|
const e = window.__openclawUiInspectorEvents || [];
|
|
window.__openclawUiInspectorEvents = [];
|
|
return e;
|
|
}"""
|
|
) or []
|
|
except Exception:
|
|
events = []
|
|
url = ""
|
|
try:
|
|
url = page.url or ""
|
|
except Exception:
|
|
url = ""
|
|
page_id = ""
|
|
opener_page_id = ""
|
|
try:
|
|
import kit_browser
|
|
|
|
page_id = kit_browser.page_id_for(page)
|
|
try:
|
|
opener = page.opener()
|
|
if opener is not None:
|
|
opener_page_id = kit_browser.page_id_for(opener)
|
|
except Exception:
|
|
pass
|
|
except Exception:
|
|
pass
|
|
return {
|
|
"url": url,
|
|
"page_id": page_id,
|
|
"opener_page_id": opener_page_id,
|
|
"events": events,
|
|
}
|
|
|
|
def _emit_dom_event(self, ev: dict[str, Any]) -> None:
|
|
event_id = str(ev.get("event_id") or "")
|
|
if event_id:
|
|
if event_id in self._seen_event_ids:
|
|
return
|
|
self._seen_event_ids.add(event_id)
|
|
self._seen_event_order.append(event_id)
|
|
if len(self._seen_event_order) > 2000:
|
|
old = self._seen_event_order.pop(0)
|
|
self._seen_event_ids.discard(old)
|
|
if self.on_raw_event:
|
|
try:
|
|
self.on_raw_event(dict(ev))
|
|
except Exception:
|
|
pass
|
|
kind = str(ev.get("kind") or "")
|
|
hit_target = dict(ev.get("hit_target") or ev.get("target") or {})
|
|
candidate_chain = list(ev.get("candidate_chain") or ev.get("ancestry") or [])
|
|
if not candidate_chain:
|
|
candidate_chain = [hit_target]
|
|
action_hint = dict(ev.get("action_target") or {})
|
|
if action_hint and action_hint != hit_target:
|
|
candidate_chain.append(action_hint)
|
|
try:
|
|
from core.policies.target_scoring import resolve_browser_target
|
|
|
|
target, target_scores = resolve_browser_target(
|
|
hit_target,
|
|
candidate_chain,
|
|
x=int(ev.get("x")) if ev.get("x") is not None else None,
|
|
y=int(ev.get("y")) if ev.get("y") is not None else None,
|
|
)
|
|
except Exception:
|
|
target = dict(ev.get("action_target") or ev.get("target") or hit_target)
|
|
target_scores = []
|
|
if candidate_chain:
|
|
target["parent_chain"] = candidate_chain
|
|
frame = {
|
|
"url": ev.get("frame_url") or "",
|
|
"name": ev.get("frame_name") or "",
|
|
"id": ev.get("frame_id") or "",
|
|
}
|
|
if kind in {"pointerdown", "click"}:
|
|
raw = RawUserEvent(
|
|
kind="click",
|
|
surface="browser",
|
|
target=target,
|
|
frame=frame,
|
|
url=str(ev.get("url") or ""),
|
|
x=int(ev.get("x")) if ev.get("x") is not None else None,
|
|
y=int(ev.get("y")) if ev.get("y") is not None else None,
|
|
meta={
|
|
"button": ev.get("button"),
|
|
"event_id": event_id,
|
|
"document_id": ev.get("document_id"),
|
|
"page_id": ev.get("page_id"),
|
|
"frame_id": ev.get("frame_id"),
|
|
"sequence": ev.get("sequence"),
|
|
"source_ts_ms": ev.get("ts"),
|
|
"hit_target": ev.get("hit_target") or {},
|
|
"action_target": ev.get("action_target") or target,
|
|
"target_scores": target_scores,
|
|
"captured_from": kind,
|
|
},
|
|
)
|
|
if kind == "pointerdown":
|
|
key = self._pointer_key(ev)
|
|
self._pending_pointerdowns[key] = {
|
|
"due": time.monotonic() + self._pointer_fallback_sec,
|
|
"event": raw,
|
|
"page_id": str(ev.get("page_id") or ""),
|
|
}
|
|
return
|
|
self._cancel_pointerdown_for_click(ev)
|
|
self.emit(raw)
|
|
elif kind == "input":
|
|
self.emit(
|
|
RawUserEvent(
|
|
kind="input",
|
|
surface="browser",
|
|
text=str(ev.get("text") or target.get("value") or ""),
|
|
target=target,
|
|
frame=frame,
|
|
url=str(ev.get("url") or ""),
|
|
meta={
|
|
"input_type": target.get("input_type") or "",
|
|
"event_id": event_id,
|
|
"document_id": ev.get("document_id"),
|
|
"page_id": ev.get("page_id"),
|
|
"frame_id": ev.get("frame_id"),
|
|
"sequence": ev.get("sequence"),
|
|
"source_ts_ms": ev.get("ts"),
|
|
"input_event": ev.get("input_event") or "input",
|
|
"is_trusted": bool(ev.get("is_trusted")),
|
|
},
|
|
)
|
|
)
|
|
elif kind == "key":
|
|
self.emit(
|
|
RawUserEvent(
|
|
kind="key",
|
|
surface="browser",
|
|
key=str(ev.get("key") or ""),
|
|
target=target,
|
|
frame=frame,
|
|
url=str(ev.get("url") or ""),
|
|
meta={
|
|
"event_id": event_id,
|
|
"document_id": ev.get("document_id"),
|
|
"page_id": ev.get("page_id"),
|
|
"frame_id": ev.get("frame_id"),
|
|
"sequence": ev.get("sequence"),
|
|
"source_ts_ms": ev.get("ts"),
|
|
},
|
|
)
|
|
)
|
|
|
|
@staticmethod
|
|
def _pointer_key(ev: dict[str, Any]) -> str:
|
|
return "|".join(
|
|
str(ev.get(key) or "")
|
|
for key in ("page_id", "document_id", "pointer_id", "button")
|
|
)
|
|
|
|
def _cancel_pointerdown_for_click(self, ev: dict[str, Any]) -> None:
|
|
exact = self._pointer_key(ev)
|
|
if exact in self._pending_pointerdowns:
|
|
self._pending_pointerdowns.pop(exact, None)
|
|
return
|
|
page_id = str(ev.get("page_id") or "")
|
|
button = str(ev.get("button") or "")
|
|
for key, item in list(self._pending_pointerdowns.items()):
|
|
event = item.get("event")
|
|
meta = getattr(event, "meta", {}) or {}
|
|
if str(meta.get("page_id") or "") == page_id and str(meta.get("button") or "") == button:
|
|
self._pending_pointerdowns.pop(key, None)
|
|
return
|
|
|
|
def _flush_pending_pointerdowns(
|
|
self, *, force: bool = False, page_id: str = ""
|
|
) -> None:
|
|
now = time.monotonic()
|
|
ready = [
|
|
key for key, item in self._pending_pointerdowns.items()
|
|
if (not page_id or item.get("page_id") == page_id)
|
|
and (force or float(item.get("due") or 0.0) <= now)
|
|
]
|
|
for key in ready:
|
|
item = self._pending_pointerdowns.pop(key, None) or {}
|
|
event = item.get("event")
|
|
if event is not None:
|
|
event.meta["pointerdown_fallback"] = True
|
|
self.emit(event)
|