Files
skill-template/tools/ui_inspector/kit_browser.py
chendelian 633ecff4e2 chore: archive tools/ui_inspector snapshot before migration removal.
Keep the last in-template copy on main so history can restore it; product continues in jiangchang-ui-recorder.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-07 11:19:14 +08:00

730 lines
23 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Inspector-owned Playwright browser session (dev-time page dump).
All Playwright sync API calls run on one dedicated worker thread
(Playwright sync is not cross-thread safe).
Dump / screenshot / recorder target the **currently visible** page
(document.visibilityState === 'visible'), not a sticky pages[0].
This keeps list ↔ detail tab switching aligned with what the user sees.
Uses jiangchang_skill_core.rpa stealth launch parts + dump_page_sync.
Profile lives under ui_inspector/_browser_profile (not account-manager).
"""
from __future__ import annotations
import atexit
import queue
import threading
import uuid
from pathlib import Path
from typing import Any, Callable, Optional
_HERE = Path(__file__).resolve().parent
_DEFAULT_PROFILE = _HERE / "_browser_profile"
_DEFAULT_START_URL = "about:blank"
_cmd_q: queue.Queue = queue.Queue()
_worker: Optional[threading.Thread] = None
_worker_lock = threading.Lock()
# Owned exclusively by worker thread:
_pw = None
_context = None
_page = None
_started = False
_bridge_installed = False
_bridge_sink: Optional[Callable[[dict[str, Any]], None]] = None
_bridge_lock = threading.Lock()
_BRIDGE_NAME = "__openclawUiInspectorEmit"
_page_ids: dict[int, str] = {}
_page_refs: dict[int, Any] = {}
_pages_by_id: dict[str, Any] = {}
_frame_ids: dict[int, str] = {}
_frame_refs: dict[int, Any] = {}
def _page_id_for(page: Any) -> str:
if not _page_is_open(page):
return ""
key = id(page)
page_id = _page_ids.get(key) if _page_refs.get(key) is page else None
if not page_id:
page_id = f"page_{uuid.uuid4().hex[:12]}"
_page_ids[key] = page_id
_page_refs[key] = page
_pages_by_id[page_id] = page
return page_id
def _frame_id_for(frame: Any, *, page_id: str = "") -> str:
if frame is None:
return ""
key = id(frame)
frame_id = _frame_ids.get(key) if _frame_refs.get(key) is frame else None
if not frame_id:
prefix = page_id or "page"
frame_id = f"{prefix}:frame_{uuid.uuid4().hex[:10]}"
_frame_ids[key] = frame_id
_frame_refs[key] = frame
return frame_id
def _page_for_id(page_id: str) -> Any:
page = _pages_by_id.get(str(page_id or ""))
if _page_is_open(page):
return page
if page_id:
_pages_by_id.pop(str(page_id), None)
return None
def kit_page_dump_available() -> bool:
try:
from jiangchang_skill_core.rpa.page_dump import dump_page_sync # noqa: F401
return True
except ImportError:
return False
def profile_dir() -> Path:
return _DEFAULT_PROFILE
def _ensure_worker() -> None:
global _worker
with _worker_lock:
if _worker is not None and _worker.is_alive():
return
def loop() -> None:
while True:
item = _cmd_q.get()
if item is None:
break
fn, args, kwargs, out_q = item
try:
out_q.put(("ok", fn(*args, **kwargs)))
except BaseException as exc:
out_q.put(("err", exc))
_worker = threading.Thread(target=loop, name="ui-inspector-pw", daemon=True)
_worker.start()
atexit.register(shutdown)
def _call_on_worker(
fn: Callable[..., Any],
*args: Any,
timeout_sec: float = 30.0,
**kwargs: Any,
) -> Any:
_ensure_worker()
out_q: queue.Queue = queue.Queue()
_cmd_q.put((fn, args, kwargs, out_q))
try:
status, payload = out_q.get(timeout=max(0.1, float(timeout_sec)))
except queue.Empty as exc:
raise RuntimeError(
f"ERROR:BROWSER_WORKER_TIMEOUT Playwright 命令超过 {timeout_sec:g} 秒未返回。"
) from exc
if status == "err":
raise payload
return payload
def _teardown_unlocked() -> None:
global _pw, _context, _page, _started, _bridge_installed, _bridge_sink
try:
if _context is not None:
_context.close()
except Exception:
pass
try:
if _pw is not None:
_pw.stop()
except Exception:
pass
_pw = None
_context = None
_page = None
_started = False
_bridge_installed = False
with _bridge_lock:
_bridge_sink = None
_page_ids.clear()
_page_refs.clear()
_pages_by_id.clear()
_frame_ids.clear()
_frame_refs.clear()
def _goto_safe(page: Any, start_url: str) -> dict[str, Any]:
url = (start_url or "").strip()
if not url or url == "about:blank":
return {"ok": True, "requested_url": url, "actual_url": page.url or ""}
try:
page.goto(url, wait_until="domcontentloaded", timeout=60_000)
except Exception as exc:
actual = ""
try:
actual = str(page.url or "")
except Exception:
pass
# Some sites keep loading after domcontentloaded timeout. A real HTTP
# page is usable, but the warning must remain visible to callers.
if actual.startswith(("http://", "https://")):
return {
"ok": True,
"requested_url": url,
"actual_url": actual,
"warning": str(exc),
}
raise RuntimeError(
f"ERROR:NAVIGATION_FAILED requested={url!r} actual={actual!r}: {exc}"
) from exc
actual = str(page.url or "")
if not actual or actual == "about:blank" or actual.startswith("about:"):
raise RuntimeError(
f"ERROR:NAVIGATION_FAILED requested={url!r} actual={actual!r}"
)
return {"ok": True, "requested_url": url, "actual_url": actual}
def _page_is_open(page: Any) -> bool:
try:
return page is not None and not page.is_closed()
except Exception:
return False
def _open_pages(context: Any) -> list[Any]:
if context is None:
return []
out: list[Any] = []
try:
pages = list(context.pages or [])
except Exception:
return []
for p in pages:
if _page_is_open(p):
_page_id_for(p)
out.append(p)
return out
def _page_visibility(page: Any) -> str:
"""Return document.visibilityState or '' on failure."""
if not _page_is_open(page):
return ""
try:
state = page.evaluate("() => document.visibilityState")
return str(state or "").strip().lower()
except Exception:
return ""
def resolve_active_page(
context: Any,
*,
preferred: Any = None,
) -> tuple[Any, dict[str, Any]]:
"""Pick the page the user is looking at (visible tab).
Returns (page, meta). Prefer visibilityState===visible; fall back to
preferred if still open, else first open page.
"""
pages = _open_pages(context)
meta: dict[str, Any] = {
"resolve": "none",
"page_count": len(pages),
"visible_count": 0,
}
if not pages:
raise RuntimeError("ERROR:NO_OPEN_PAGE Playwright 会话中没有可用标签页。")
visible: list[Any] = []
for p in pages:
if _page_visibility(p) == "visible":
visible.append(p)
meta["visible_count"] = len(visible)
if len(visible) == 1:
meta["resolve"] = "visibility"
return visible[0], meta
if len(visible) > 1:
# Multiple windows each with a visible tab — prefer preferred if among them
if preferred is not None and preferred in visible:
meta["resolve"] = "visibility_preferred"
return preferred, meta
meta["resolve"] = "visibility_first"
return visible[0], meta
if preferred is not None and _page_is_open(preferred) and preferred in pages:
meta["resolve"] = "preferred_fallback"
return preferred, meta
meta["resolve"] = "first_open"
return pages[0], meta
def _sync_active_page(*, update_global: bool = True) -> tuple[Any, dict[str, Any]]:
"""Resolve visible page; optionally assign to module ``_page``."""
global _page
if _context is None:
raise RuntimeError("ERROR:NO_BROWSER_SESSION Playwright 会话未启动。")
page, meta = resolve_active_page(_context, preferred=_page)
if update_global:
_page = page
try:
meta["url"] = page.url or ""
meta["title"] = page.title() or ""
except Exception:
meta["url"] = ""
meta["title"] = ""
meta["page_id"] = _page_id_for(page)
return page, meta
def _resolve_capture_page(page_id: str = "") -> tuple[Any, dict[str, Any]]:
global _page
requested = str(page_id or "")
if requested:
page = _page_for_id(requested)
if page is not None:
_page = page
meta = {
"resolve": "event_page_id",
"page_id": requested,
"page_count": len(_open_pages(_context)),
"visible_count": None,
}
try:
meta["url"] = page.url or ""
meta["title"] = page.title() or ""
except Exception:
meta["url"] = ""
meta["title"] = ""
return page, meta
raise RuntimeError(f"ERROR:EVENT_PAGE_CLOSED page_id={requested}")
return _sync_active_page(update_global=True)
def _ensure_session_worker(
*,
start_url: str = _DEFAULT_START_URL,
force_goto: bool = False,
) -> dict[str, Any]:
global _pw, _context, _page, _started
want = (start_url or "").strip() or _DEFAULT_START_URL
if _started and _context is not None:
try:
navigation = None
pages = _open_pages(_context)
if not pages:
_teardown_unlocked()
else:
# Start-flow navigation: prefer first open tab (stable home),
# not whatever detail tab happens to be visible.
if force_goto and want and want != "about:blank":
home = pages[0]
cur = ""
try:
cur = home.url or ""
except Exception:
cur = ""
if cur.rstrip("/") != want.rstrip("/"):
navigation = _goto_safe(home, want)
_page = home
else:
_page, _meta = _sync_active_page(update_global=True)
if want and want != "about:blank" and (
(_page.url or "") in {"", "about:blank"}
or str(_page.url or "").startswith("about:")
):
navigation = _goto_safe(_page, want)
_page, _meta = _sync_active_page(update_global=True)
try:
live_url = _page.url or ""
except Exception:
live_url = ""
return {
"ok": True,
"reused": True,
"url": live_url,
"profile_dir": str(profile_dir()),
"page_count": len(_open_pages(_context)),
"navigation": navigation,
}
except Exception:
_teardown_unlocked()
if not kit_page_dump_available():
raise RuntimeError(
"ERROR:KIT_PAGE_DUMP_MISSING 需要 jiangchang-platform-kit>=1.2.11 "
"(jiangchang_skill_core.rpa.page_dump)。请升级 kit 后重试。"
)
try:
from playwright.sync_api import sync_playwright
except ImportError as exc:
raise RuntimeError(
"ERROR:MISSING_PLAYWRIGHT 未安装 playwright。"
"请在与技能相同的 Python 环境中安装后重试。"
) from exc
from jiangchang_skill_core.runtime_env import find_chrome_executable
from jiangchang_skill_core.rpa.stealth import (
STEALTH_INIT_SCRIPT,
persistent_context_launch_parts,
stealth_enabled,
)
chrome = find_chrome_executable()
if not chrome:
raise RuntimeError(
"ERROR:MISSING_BROWSER 未检测到 Chrome 或 Edge,请安装后重试。"
)
prof = profile_dir()
prof.mkdir(parents=True, exist_ok=True)
args, ignore = persistent_context_launch_parts(extra_args=["--start-maximized"])
launch_kwargs: dict[str, Any] = dict(
user_data_dir=str(prof),
headless=False,
executable_path=chrome,
locale="zh-CN",
no_viewport=True,
args=args,
)
if ignore is not None:
launch_kwargs["ignore_default_args"] = ignore
_pw = sync_playwright().start()
try:
_context = _pw.chromium.launch_persistent_context(**launch_kwargs)
if stealth_enabled():
_context.add_init_script(STEALTH_INIT_SCRIPT)
if _context.pages:
_page = _context.pages[0]
else:
_page = _context.new_page()
navigation = None
if want and want != "about:blank":
navigation = _goto_safe(_page, want)
elif want and (
_page.url in {"", "about:blank"} or str(_page.url).startswith("about:")
):
navigation = _goto_safe(_page, want)
_started = True
return {
"ok": True,
"reused": False,
"url": _page.url,
"profile_dir": str(prof),
"page_count": len(_open_pages(_context)),
"navigation": navigation,
}
except Exception:
_teardown_unlocked()
raise
def _dump_worker(*, max_nodes: int, locatable_limit: int, page_id: str = "") -> dict[str, Any]:
from jiangchang_skill_core.rpa.page_dump import dump_page_sync
_ensure_session_worker()
page, meta = _resolve_capture_page(page_id)
result = dump_page_sync(page, max_nodes=max_nodes, locatable_limit=locatable_limit)
stats = dict(result.get("stats") or {})
stats["active_page"] = meta
result["stats"] = stats
return result
def _screenshot_worker(*, page_id: str = "") -> bytes:
_ensure_session_worker()
page, _meta = _resolve_capture_page(page_id)
return page.screenshot(type="png", full_page=False)
def _session_info_worker() -> dict[str, Any]:
if not _started or _context is None:
return {"active": False}
try:
page, meta = _sync_active_page(update_global=True)
return {
"active": True,
"url": page.url,
"title": page.title(),
"profile_dir": str(profile_dir()),
"page_count": meta.get("page_count"),
"resolve": meta.get("resolve"),
"page_id": meta.get("page_id"),
}
except Exception as exc:
return {"active": False, "error": str(exc)}
def _dispatch_bridge_event(source: Any, payload: Any) -> None:
if not isinstance(payload, dict):
return
item = dict(payload)
try:
page = source.get("page") if isinstance(source, dict) else None
frame = source.get("frame") if isinstance(source, dict) else None
page_id = _page_id_for(page)
item["page_id"] = page_id
item["frame_id"] = _frame_id_for(frame, page_id=page_id)
except Exception:
pass
with _bridge_lock:
sink = _bridge_sink
if sink is not None:
try:
sink(item)
except Exception:
pass
def _install_event_bridge_worker(
*, init_script: str, sink: Callable[[dict[str, Any]], None]
) -> dict[str, Any]:
global _bridge_installed, _bridge_sink
_ensure_session_worker()
assert _context is not None
with _bridge_lock:
_bridge_sink = sink
if not _bridge_installed:
_context.expose_binding(_BRIDGE_NAME, _dispatch_bridge_event)
_bridge_installed = True
_context.add_init_script(init_script)
installed = 0
for page in _open_pages(_context):
try:
page.evaluate(init_script)
installed += 1
except Exception:
pass
return {"ok": True, "binding": _BRIDGE_NAME, "installed_pages": installed}
def install_event_bridge(
init_script: str, sink: Callable[[dict[str, Any]], None]
) -> dict[str, Any]:
"""Install a context-level recorder bridge before user actions begin."""
return _call_on_worker(
_install_event_bridge_worker,
init_script=init_script,
sink=sink,
timeout_sec=15.0,
)
def clear_event_bridge() -> None:
global _bridge_sink
with _bridge_lock:
_bridge_sink = None
def ensure_session(
*,
start_url: str = _DEFAULT_START_URL,
force_goto: bool = False,
) -> dict[str, Any]:
"""Start or reuse headed persistent context; return status dict.
force_goto=True: navigate to start_url even when session is reused
(used when「开始流程」带起始 URL).
"""
return _call_on_worker(
_ensure_session_worker,
start_url=start_url,
force_goto=force_goto,
timeout_sec=75.0,
)
def session_info() -> dict[str, Any]:
return _call_on_worker(_session_info_worker, timeout_sec=10.0)
def dump_current_page(
*, max_nodes: int = 8000, locatable_limit: int = 300, page_id: str = ""
) -> dict[str, Any]:
"""Dump an event-origin page, or resolve the active page as fallback."""
return _call_on_worker(
_dump_worker,
max_nodes=max_nodes,
locatable_limit=locatable_limit,
page_id=page_id,
timeout_sec=30.0,
)
def screenshot_png(*, page_id: str = "") -> bytes:
return _call_on_worker(_screenshot_worker, page_id=page_id, timeout_sec=20.0)
def page_id_for(page: Any) -> str:
"""Return the stable id for a Page while already on the browser worker."""
return _page_id_for(page)
def _validate_element_worker(
*, page_id: str, element: dict[str, Any], frame: Optional[dict[str, Any]] = None
) -> dict[str, int]:
page, _meta = _resolve_capture_page(page_id)
scope: Any = page
frame_meta = dict(frame or {})
frame_url = str(frame_meta.get("url") or "")
frame_name = str(frame_meta.get("name") or "")
if frame_url or frame_name:
try:
for candidate in page.frames:
if frame_name and candidate.name == frame_name:
scope = candidate
break
if frame_url and candidate.url == frame_url:
scope = candidate
break
except Exception:
pass
el = dict(element or {})
role = str(el.get("role") or "").strip()
name = str(el.get("name") or "").strip()
test_id = str(el.get("test_id") or "").strip()
label = str(el.get("label") or "").strip()
placeholder = str(el.get("placeholder") or "").strip()
name_attr = str(el.get("name_attr") or "").strip()
href = str(el.get("href") or "").strip()
text_value = str(el.get("text") or "").strip()
css_id = str(el.get("id") or "").strip()
css_path = str(el.get("css") or el.get("css_path") or "").strip()
stable_css = str(el.get("stable_css") or "").strip()
semantic_role = str(el.get("semantic_role") or "").strip()
semantic_name = str(el.get("semantic_name") or "").strip()
counts: dict[str, int] = {}
def count(strategy: str, build: Callable[[], Any]) -> None:
try:
counts[strategy] = int(build().count())
except Exception:
pass
def count_actionable(strategy: str, build: Callable[[], Any]) -> None:
try:
locator = build()
total = min(int(locator.count()), 50)
actionable = 0
for index in range(total):
item = locator.nth(index)
if item.is_visible() and item.is_enabled():
actionable += 1
counts[strategy] = actionable
except Exception:
pass
if role and name:
# Playwright accessible names normalize whitespace (SVG/icon buttons are
# a common example), while DOM innerText may not. Non-exact still
# yields a trustworthy uniqueness count and mirrors getByRole defaults.
count("role", lambda: scope.get_by_role(role, name=name, exact=False))
elif role and bool(el.get("contenteditable")):
count("role", lambda: scope.get_by_role(role))
if semantic_role and semantic_name:
count_actionable(
"semantic_role",
lambda: scope.get_by_role(
semantic_role, name=semantic_name, exact=False
),
)
if test_id:
count("test_id", lambda: scope.get_by_test_id(test_id))
if label:
count("label", lambda: scope.get_by_label(label, exact=True))
if placeholder:
count("placeholder", lambda: scope.get_by_placeholder(placeholder, exact=True))
if name_attr:
escaped_name = name_attr.replace("\\", "\\\\").replace('"', '\\"')
count("name_attr", lambda: scope.locator(f'[name="{escaped_name}"]'))
if href:
escaped_href = href.replace("\\", "\\\\").replace('"', '\\"')
count("href", lambda: scope.locator(f'a[href="{escaped_href}"]'))
if text_value and len(text_value) <= 80:
count("text", lambda: scope.get_by_text(text_value, exact=True))
if css_id and not css_id[0].isdigit():
escaped = css_id.replace("\\", "\\\\").replace('"', '\\"')
count("css_id", lambda: scope.locator(f'[id="{escaped}"]'))
if css_path:
count("css", lambda: scope.locator(css_path))
if stable_css:
count("stable_css", lambda: scope.locator(stable_css))
return counts
def validate_element_on_page(
*, page_id: str, element: dict[str, Any], frame: Optional[dict[str, Any]] = None
) -> dict[str, int]:
"""Count locator candidates against the exact live event-origin page."""
if not page_id:
return {}
return _call_on_worker(
_validate_element_worker,
page_id=page_id,
element=element,
frame=frame,
timeout_sec=10.0,
)
def call_on_page(fn: Callable[[Any], Any]) -> Any:
"""Run ``fn(page)`` on the **visible** Playwright page (worker thread)."""
def _run() -> Any:
_ensure_session_worker()
page, _meta = _sync_active_page(update_global=True)
return fn(page)
return _call_on_worker(_run, timeout_sec=30.0)
def call_on_each_open_page(fn: Callable[[Any], Any]) -> list[Any]:
"""Run ``fn(page)`` for every open tab; returns list of per-page results."""
def _run() -> list[Any]:
_ensure_session_worker()
assert _context is not None
pages = _open_pages(_context)
# Keep _page synced to visible for dump consistency
try:
_sync_active_page(update_global=True)
except Exception:
pass
results: list[Any] = []
for p in pages:
try:
results.append(fn(p))
except Exception as exc:
results.append({"error": str(exc)})
return results
return _call_on_worker(_run, timeout_sec=20.0)
def shutdown() -> None:
def _stop() -> None:
_teardown_unlocked()
try:
_call_on_worker(_stop, timeout_sec=5.0)
except Exception:
pass
try:
_cmd_q.put(None)
except Exception:
pass