chendelian
633ecff4e2
Keep the last in-template copy on main so history can restore it; product continues in jiangchang-ui-recorder. Co-authored-by: Cursor <cursoragent@cursor.com>
409 lines
17 KiB
Python
409 lines
17 KiB
Python
"""Flow session glue: v2 ActionRecordingManager + surface recorders + v1 dual-write."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import queue
|
|
import threading
|
|
import uuid
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Optional
|
|
|
|
import artifacts as art
|
|
from core.pipeline.event_journal import EventJournal
|
|
from evidence.coordinator import EvidenceCoordinator
|
|
from recorder.base import RawUserEvent
|
|
from recorder.manager import ActionRecordingManager
|
|
|
|
|
|
class FlowSession:
|
|
def __init__(
|
|
self,
|
|
flow_dir: Path,
|
|
*,
|
|
flow_name: str,
|
|
surface: str,
|
|
recording_mode: str = "auto",
|
|
dump_fn: Optional[Callable[[], Any]] = None,
|
|
write_v1_step: Optional[Callable[..., Any]] = None,
|
|
exclude_hwnds: Optional[Callable[[], set[int]]] = None,
|
|
on_action_update: Optional[Callable[[Any, str], None]] = None,
|
|
stability_timeout_sec: float = 3.0,
|
|
) -> None:
|
|
self.flow_dir = Path(flow_dir)
|
|
self.surface = surface
|
|
self.recording_mode = recording_mode
|
|
self._dump_fn = dump_fn
|
|
self.exclude_hwnds = exclude_hwnds or (lambda: set())
|
|
self.manager = ActionRecordingManager(
|
|
self.flow_dir,
|
|
flow_name=flow_name,
|
|
flow_id=self.flow_dir.name,
|
|
surface=surface,
|
|
recording_mode=recording_mode,
|
|
dump_fn=dump_fn,
|
|
write_v1_step=write_v1_step,
|
|
stability_timeout_sec=stability_timeout_sec,
|
|
defer_action_evidence=(
|
|
surface in {"browser", "windows"}
|
|
and recording_mode != "legacy_interval"
|
|
),
|
|
on_action_update=on_action_update,
|
|
)
|
|
self.journal = EventJournal(self.flow_dir)
|
|
self._evidence = (
|
|
EvidenceCoordinator(self.manager, dump_fn, on_dump=self._on_evidence_dump)
|
|
if surface in {"browser", "windows"}
|
|
and dump_fn is not None
|
|
and recording_mode != "legacy_interval"
|
|
else None
|
|
)
|
|
if self._evidence is not None:
|
|
self.manager.on_action = self._evidence.schedule
|
|
self._recorder = None
|
|
self._recorder_health: dict[str, Any] = {}
|
|
self._event_queue: queue.Queue[tuple[str, Any]] = queue.Queue(maxsize=512)
|
|
self._event_thread: Optional[threading.Thread] = None
|
|
self._event_stop = threading.Event()
|
|
self._stats_lock = threading.Lock()
|
|
self._stats_cache: dict[str, Any] = {
|
|
"recorder_status": "idle",
|
|
"surface_recorder": "off",
|
|
"state_count": 0,
|
|
"observation_count": 0,
|
|
"action_count": 0,
|
|
"element_count": 0,
|
|
"queued_events": 0,
|
|
"dropped_events": 0,
|
|
}
|
|
self._dropped_events = 0
|
|
|
|
def start(self, *, entry_dump: Optional[Any] = None) -> None:
|
|
if self.recording_mode == "legacy_interval":
|
|
self.manager.start(entry_dump=entry_dump)
|
|
self._refresh_stats()
|
|
return
|
|
# Start observation before entry persistence. Events are journaled and
|
|
# buffered until manager.start() completes, eliminating startup loss.
|
|
self._start_surface_recorder()
|
|
if self.surface == "windows" and entry_dump is not None:
|
|
self._on_evidence_dump(entry_dump)
|
|
if self._evidence is not None:
|
|
self._evidence.start()
|
|
try:
|
|
self.manager.start(entry_dump=entry_dump)
|
|
except Exception:
|
|
if self._recorder:
|
|
self._recorder.stop()
|
|
self._recorder = None
|
|
if self._evidence is not None:
|
|
self._evidence.stop(timeout=3.0)
|
|
raise
|
|
self._start_event_consumer()
|
|
if self.surface == "windows" and self._evidence is not None:
|
|
# Warm/enrich UIA immediately, but on the evidence worker. Raw
|
|
# recording and Action reduction remain fully independent.
|
|
self._evidence.prime()
|
|
self._refresh_stats()
|
|
|
|
def _submit(self, kind: str, payload: Any = None) -> bool:
|
|
try:
|
|
if kind == "stop":
|
|
self._event_queue.put((kind, payload), timeout=2.0)
|
|
else:
|
|
self._event_queue.put_nowait((kind, payload))
|
|
return True
|
|
except queue.Full:
|
|
self._dropped_events += 1
|
|
self.manager.last_warning = (
|
|
f"recorder queue full; dropped={self._dropped_events}"
|
|
)
|
|
self._refresh_stats()
|
|
return False
|
|
|
|
def _start_event_consumer(self) -> None:
|
|
if self._event_thread and self._event_thread.is_alive():
|
|
return
|
|
self._event_stop.clear()
|
|
|
|
def consume() -> None:
|
|
while not self._event_stop.is_set():
|
|
try:
|
|
kind, payload = self._event_queue.get(timeout=0.2)
|
|
except queue.Empty:
|
|
continue
|
|
try:
|
|
if kind == "stop":
|
|
break
|
|
if kind == "event":
|
|
self.manager.handle_raw_event(payload)
|
|
elif kind == "heartbeat":
|
|
self.manager.heartbeat()
|
|
elif kind == "pause":
|
|
self.manager.pause()
|
|
elif kind == "resume":
|
|
self.manager.resume()
|
|
except Exception as exc:
|
|
self.manager.last_warning = str(exc)
|
|
finally:
|
|
self._event_queue.task_done()
|
|
self._refresh_stats()
|
|
self._event_stop.set()
|
|
|
|
self._event_thread = threading.Thread(
|
|
target=consume,
|
|
name="ui-inspector-action-consumer",
|
|
daemon=True,
|
|
)
|
|
self._event_thread.start()
|
|
|
|
def _start_surface_recorder(self) -> None:
|
|
def on_event(ev: RawUserEvent) -> None:
|
|
# Preserve the observed order and bound memory/thread growth.
|
|
if self.surface == "windows" and not (ev.meta or {}).get("event_id"):
|
|
ev.meta["event_id"] = f"windows:{uuid.uuid4().hex}"
|
|
self.journal.append(ev, channel="semantic")
|
|
self._submit("event", ev)
|
|
|
|
def on_raw_event(payload: dict[str, Any]) -> None:
|
|
self.journal.append(payload, channel="browser_dom")
|
|
|
|
if self.surface == "browser":
|
|
from surfaces.browser import BrowserRecorder
|
|
|
|
self._recorder = BrowserRecorder(
|
|
on_event=on_event, on_raw_event=on_raw_event
|
|
)
|
|
else:
|
|
from surfaces.windows import WindowsRecorder
|
|
|
|
self._recorder = WindowsRecorder(
|
|
on_event=on_event,
|
|
exclude_hwnds=self.exclude_hwnds,
|
|
)
|
|
self._seed_windows_element_atlas()
|
|
self._recorder.start()
|
|
|
|
def _on_evidence_dump(self, dump: Any) -> None:
|
|
recorder = self._recorder
|
|
if self.surface != "windows" or recorder is None or not hasattr(recorder, "learn_elements"):
|
|
return
|
|
try:
|
|
recorder.learn_elements(
|
|
list(getattr(dump, "locatable", None) or []),
|
|
window=dict(getattr(dump, "window", None) or {}),
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
def _seed_windows_element_atlas(self) -> None:
|
|
"""Reuse recent captures as the event-time element repository.
|
|
|
|
Values are discarded by ``learn_elements``; only stable names and
|
|
geometry are retained. This lets a new recording identify custom
|
|
editors before their slow accessibility provider completes a new dump.
|
|
"""
|
|
recorder = self._recorder
|
|
if recorder is None or not hasattr(recorder, "learn_elements"):
|
|
return
|
|
try:
|
|
siblings = sorted(
|
|
(
|
|
path for path in self.flow_dir.parent.iterdir()
|
|
if path.is_dir() and path != self.flow_dir
|
|
),
|
|
key=lambda path: path.stat().st_mtime,
|
|
reverse=True,
|
|
)[:8]
|
|
except Exception:
|
|
return
|
|
learned_classes: set[str] = set()
|
|
for sibling in siblings:
|
|
observations = sibling / "observations"
|
|
if not observations.is_dir():
|
|
continue
|
|
try:
|
|
metas = sorted(
|
|
observations.glob("*/meta.json"),
|
|
key=lambda path: path.stat().st_mtime,
|
|
reverse=True,
|
|
)
|
|
except Exception:
|
|
continue
|
|
for meta_path in metas:
|
|
try:
|
|
import json
|
|
|
|
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
|
if str(meta.get("surface") or "") != "windows":
|
|
continue
|
|
selection = dict(meta.get("selection") or {})
|
|
class_name = str(
|
|
selection.get("chosen_class")
|
|
or selection.get("event_class")
|
|
or ""
|
|
)
|
|
if not class_name or class_name in learned_classes:
|
|
continue
|
|
locatable_path = meta_path.parent / "locatable.json"
|
|
rows = json.loads(locatable_path.read_text(encoding="utf-8"))
|
|
recorder.learn_elements(
|
|
rows,
|
|
window={
|
|
"class_name": class_name,
|
|
"hwnd": int(selection.get("chosen_hwnd") or 0),
|
|
"pid": int(selection.get("chosen_pid") or 0),
|
|
"title": str(selection.get("event_title") or ""),
|
|
},
|
|
)
|
|
learned_classes.add(class_name)
|
|
except Exception:
|
|
continue
|
|
if "WeWorkWindow" in learned_classes and "SearchResultWindow2" in learned_classes:
|
|
break
|
|
|
|
def pause(self) -> None:
|
|
if self._recorder:
|
|
self._recorder.pause()
|
|
self._submit("pause")
|
|
|
|
def resume(self) -> None:
|
|
if self._recorder:
|
|
self._recorder.resume()
|
|
self._submit("resume")
|
|
|
|
def stop(self) -> None:
|
|
recorder_stop_error: Optional[BaseException] = None
|
|
if self._recorder:
|
|
try:
|
|
self._recorder.stop()
|
|
except Exception as exc:
|
|
recorder_stop_error = exc
|
|
if hasattr(self._recorder, "health"):
|
|
try:
|
|
self._recorder_health = self._recorder.health()
|
|
except Exception as exc:
|
|
self._recorder_health = {"last_error": str(exc)}
|
|
self._recorder = None
|
|
if self._event_thread and self._event_thread.is_alive():
|
|
self._submit("stop")
|
|
self._event_thread.join(timeout=35.0)
|
|
if self._event_thread.is_alive():
|
|
self.manager.last_warning = "event consumer did not stop within 35s"
|
|
raise RuntimeError(self.manager.last_warning)
|
|
else:
|
|
self._event_thread = None
|
|
self._event_stop.set()
|
|
if self._evidence is not None:
|
|
self._evidence.stop(timeout=3.0)
|
|
self.manager.stop()
|
|
# Point flow.json v2 fields (already present from create_flow).
|
|
try:
|
|
flow = art.load_flow(self.flow_dir)
|
|
v2 = dict(flow.get("v2") or {})
|
|
v2["action_count"] = len(self.manager.action_flow.actions)
|
|
v2["state_count"] = len(self.manager.action_flow.states)
|
|
v2["observation_count"] = len(
|
|
self.manager.snapshots.observation_ids()
|
|
)
|
|
v2["element_count"] = len(self.manager.catalog.catalog.elements)
|
|
v2["raw_event_journal"] = "journal/raw_events.jsonl"
|
|
v2["pages"] = "pages.json"
|
|
v2["diagnostics_dir"] = "diagnostics/"
|
|
v2["validation"] = "diagnostics/validation.json"
|
|
v2["replay_report"] = "diagnostics/replay_report.json"
|
|
flow["v2"] = v2
|
|
art.save_flow(self.flow_dir, flow)
|
|
except Exception:
|
|
pass
|
|
self._write_diagnostics()
|
|
self._refresh_stats()
|
|
if recorder_stop_error is not None:
|
|
self.manager.last_warning = str(recorder_stop_error)
|
|
raise RuntimeError(str(recorder_stop_error)) from recorder_stop_error
|
|
|
|
def checkpoint(self) -> Any:
|
|
return self.manager.record_checkpoint()
|
|
|
|
def heartbeat(self) -> None:
|
|
self._submit("heartbeat")
|
|
|
|
def stats(self) -> dict[str, Any]:
|
|
with self._stats_lock:
|
|
return dict(self._stats_cache)
|
|
|
|
def _refresh_stats(self) -> None:
|
|
st = self.manager.stats()
|
|
st["surface_recorder"] = (
|
|
self._recorder.status if self._recorder else "off"
|
|
)
|
|
st["queued_events"] = self._event_queue.qsize()
|
|
st["dropped_events"] = self._dropped_events
|
|
if self._recorder is not None and hasattr(self._recorder, "health"):
|
|
try:
|
|
st["input_backend"] = self._recorder.health()
|
|
except Exception as exc:
|
|
st["input_backend"] = {"last_error": str(exc)}
|
|
elif self._recorder_health:
|
|
st["input_backend"] = dict(self._recorder_health)
|
|
if self._evidence is not None:
|
|
st["evidence_capture_count"] = self._evidence.capture_count
|
|
st["evidence_dropped"] = self._evidence.dropped
|
|
st["evidence_queued"] = self._evidence._queue.qsize()
|
|
st["transient_evidence_queued"] = self._evidence._urgent_queue.qsize()
|
|
st["evidence_failures"] = self._evidence.failure_count
|
|
with self._stats_lock:
|
|
self._stats_cache = st
|
|
|
|
def _write_diagnostics(self) -> None:
|
|
pages: dict[str, dict[str, Any]] = {}
|
|
lags: list[int] = []
|
|
for action in self.manager._actions_by_id.values():
|
|
raw = dict(action.raw_event or {})
|
|
meta = dict(raw.get("meta") or {})
|
|
page_id = str(meta.get("page_id") or "")
|
|
if page_id:
|
|
page = pages.setdefault(page_id, {"page_id": page_id, "urls": [], "document_ids": []})
|
|
url = str(raw.get("url") or (action.input or {}).get("url") or "")
|
|
document_id = str(meta.get("document_id") or "")
|
|
if url and url not in page["urls"]:
|
|
page["urls"].append(url)
|
|
if document_id and document_id not in page["document_ids"]:
|
|
page["document_ids"].append(document_id)
|
|
lag = (action.evidence or {}).get("evidence_lag_ms")
|
|
if isinstance(lag, int):
|
|
lags.append(lag)
|
|
diagnostics = self.flow_dir / "diagnostics"
|
|
diagnostics.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
art.write_json(self.flow_dir / "pages.json", {
|
|
"schema": "openclaw.ui_inspector.pages.v1",
|
|
"pages": list(pages.values()),
|
|
})
|
|
recorder_health = dict(self._recorder_health)
|
|
if self._recorder is not None and hasattr(self._recorder, "health"):
|
|
try:
|
|
recorder_health = self._recorder.health()
|
|
except Exception as exc:
|
|
recorder_health = {"last_error": str(exc)}
|
|
art.write_json(diagnostics / "metrics.json", {
|
|
"schema": "openclaw.ui_inspector.metrics.v1",
|
|
"raw_event_count": self.journal._sequence,
|
|
"action_count": len(self.manager.action_flow.actions),
|
|
"state_count": len(self.manager.action_flow.states),
|
|
"observation_count": len(self.manager.snapshots.observation_ids()),
|
|
"element_count": len(self.manager.catalog.catalog.elements),
|
|
"event_queue_dropped": self._dropped_events,
|
|
"evidence_queue_dropped": self._evidence.dropped if self._evidence else 0,
|
|
"evidence_capture_count": self._evidence.capture_count if self._evidence else 0,
|
|
"evidence_failure_count": self._evidence.failure_count if self._evidence else 0,
|
|
"evidence_lag_ms_max": max(lags) if lags else None,
|
|
"evidence_lag_ms_average": round(sum(lags) / len(lags), 2) if lags else None,
|
|
"input_backend": recorder_health,
|
|
})
|
|
from diagnostics import validate_capture, write_replay_preflight
|
|
|
|
art.write_json(diagnostics / "validation.json", validate_capture(self.flow_dir))
|
|
write_replay_preflight(self.flow_dir)
|
|
except Exception as exc:
|
|
self.manager.last_warning = f"diagnostics write failed: {exc}"
|