"""Flow session glue: v2 ActionRecordingManager + surface recorders + v1 dual-write.""" from __future__ import annotations import queue import threading import uuid from pathlib import Path from typing import Any, Callable, Optional import artifacts as art from core.pipeline.event_journal import EventJournal from evidence.coordinator import EvidenceCoordinator from recorder.base import RawUserEvent from recorder.manager import ActionRecordingManager class FlowSession: def __init__( self, flow_dir: Path, *, flow_name: str, surface: str, recording_mode: str = "auto", dump_fn: Optional[Callable[[], Any]] = None, write_v1_step: Optional[Callable[..., Any]] = None, exclude_hwnds: Optional[Callable[[], set[int]]] = None, on_action_update: Optional[Callable[[Any, str], None]] = None, stability_timeout_sec: float = 3.0, ) -> None: self.flow_dir = Path(flow_dir) self.surface = surface self.recording_mode = recording_mode self._dump_fn = dump_fn self.exclude_hwnds = exclude_hwnds or (lambda: set()) self.manager = ActionRecordingManager( self.flow_dir, flow_name=flow_name, flow_id=self.flow_dir.name, surface=surface, recording_mode=recording_mode, dump_fn=dump_fn, write_v1_step=write_v1_step, stability_timeout_sec=stability_timeout_sec, defer_action_evidence=( surface in {"browser", "windows"} and recording_mode != "legacy_interval" ), on_action_update=on_action_update, ) self.journal = EventJournal(self.flow_dir) self._evidence = ( EvidenceCoordinator(self.manager, dump_fn, on_dump=self._on_evidence_dump) if surface in {"browser", "windows"} and dump_fn is not None and recording_mode != "legacy_interval" else None ) if self._evidence is not None: self.manager.on_action = self._evidence.schedule self._recorder = None self._recorder_health: dict[str, Any] = {} self._event_queue: queue.Queue[tuple[str, Any]] = queue.Queue(maxsize=512) self._event_thread: Optional[threading.Thread] = None self._event_stop = threading.Event() self._stats_lock = threading.Lock() self._stats_cache: dict[str, Any] = { "recorder_status": "idle", "surface_recorder": "off", "state_count": 0, "observation_count": 0, "action_count": 0, "element_count": 0, "queued_events": 0, "dropped_events": 0, } self._dropped_events = 0 def start(self, *, entry_dump: Optional[Any] = None) -> None: if self.recording_mode == "legacy_interval": self.manager.start(entry_dump=entry_dump) self._refresh_stats() return # Start observation before entry persistence. Events are journaled and # buffered until manager.start() completes, eliminating startup loss. self._start_surface_recorder() if self.surface == "windows" and entry_dump is not None: self._on_evidence_dump(entry_dump) if self._evidence is not None: self._evidence.start() try: self.manager.start(entry_dump=entry_dump) except Exception: if self._recorder: self._recorder.stop() self._recorder = None if self._evidence is not None: self._evidence.stop(timeout=3.0) raise self._start_event_consumer() if self.surface == "windows" and self._evidence is not None: # Warm/enrich UIA immediately, but on the evidence worker. Raw # recording and Action reduction remain fully independent. self._evidence.prime() self._refresh_stats() def _submit(self, kind: str, payload: Any = None) -> bool: try: if kind == "stop": self._event_queue.put((kind, payload), timeout=2.0) else: self._event_queue.put_nowait((kind, payload)) return True except queue.Full: self._dropped_events += 1 self.manager.last_warning = ( f"recorder queue full; dropped={self._dropped_events}" ) self._refresh_stats() return False def _start_event_consumer(self) -> None: if self._event_thread and self._event_thread.is_alive(): return self._event_stop.clear() def consume() -> None: while not self._event_stop.is_set(): try: kind, payload = self._event_queue.get(timeout=0.2) except queue.Empty: continue try: if kind == "stop": break if kind == "event": self.manager.handle_raw_event(payload) elif kind == "heartbeat": self.manager.heartbeat() elif kind == "pause": self.manager.pause() elif kind == "resume": self.manager.resume() except Exception as exc: self.manager.last_warning = str(exc) finally: self._event_queue.task_done() self._refresh_stats() self._event_stop.set() self._event_thread = threading.Thread( target=consume, name="ui-inspector-action-consumer", daemon=True, ) self._event_thread.start() def _start_surface_recorder(self) -> None: def on_event(ev: RawUserEvent) -> None: # Preserve the observed order and bound memory/thread growth. if self.surface == "windows" and not (ev.meta or {}).get("event_id"): ev.meta["event_id"] = f"windows:{uuid.uuid4().hex}" self.journal.append(ev, channel="semantic") self._submit("event", ev) def on_raw_event(payload: dict[str, Any]) -> None: self.journal.append(payload, channel="browser_dom") if self.surface == "browser": from surfaces.browser import BrowserRecorder self._recorder = BrowserRecorder( on_event=on_event, on_raw_event=on_raw_event ) else: from surfaces.windows import WindowsRecorder self._recorder = WindowsRecorder( on_event=on_event, exclude_hwnds=self.exclude_hwnds, ) self._seed_windows_element_atlas() self._recorder.start() def _on_evidence_dump(self, dump: Any) -> None: recorder = self._recorder if self.surface != "windows" or recorder is None or not hasattr(recorder, "learn_elements"): return try: recorder.learn_elements( list(getattr(dump, "locatable", None) or []), window=dict(getattr(dump, "window", None) or {}), ) except Exception: pass def _seed_windows_element_atlas(self) -> None: """Reuse recent captures as the event-time element repository. Values are discarded by ``learn_elements``; only stable names and geometry are retained. This lets a new recording identify custom editors before their slow accessibility provider completes a new dump. """ recorder = self._recorder if recorder is None or not hasattr(recorder, "learn_elements"): return try: siblings = sorted( ( path for path in self.flow_dir.parent.iterdir() if path.is_dir() and path != self.flow_dir ), key=lambda path: path.stat().st_mtime, reverse=True, )[:8] except Exception: return learned_classes: set[str] = set() for sibling in siblings: observations = sibling / "observations" if not observations.is_dir(): continue try: metas = sorted( observations.glob("*/meta.json"), key=lambda path: path.stat().st_mtime, reverse=True, ) except Exception: continue for meta_path in metas: try: import json meta = json.loads(meta_path.read_text(encoding="utf-8")) if str(meta.get("surface") or "") != "windows": continue selection = dict(meta.get("selection") or {}) class_name = str( selection.get("chosen_class") or selection.get("event_class") or "" ) if not class_name or class_name in learned_classes: continue locatable_path = meta_path.parent / "locatable.json" rows = json.loads(locatable_path.read_text(encoding="utf-8")) recorder.learn_elements( rows, window={ "class_name": class_name, "hwnd": int(selection.get("chosen_hwnd") or 0), "pid": int(selection.get("chosen_pid") or 0), "title": str(selection.get("event_title") or ""), }, ) learned_classes.add(class_name) except Exception: continue if "WeWorkWindow" in learned_classes and "SearchResultWindow2" in learned_classes: break def pause(self) -> None: if self._recorder: self._recorder.pause() self._submit("pause") def resume(self) -> None: if self._recorder: self._recorder.resume() self._submit("resume") def stop(self) -> None: recorder_stop_error: Optional[BaseException] = None if self._recorder: try: self._recorder.stop() except Exception as exc: recorder_stop_error = exc if hasattr(self._recorder, "health"): try: self._recorder_health = self._recorder.health() except Exception as exc: self._recorder_health = {"last_error": str(exc)} self._recorder = None if self._event_thread and self._event_thread.is_alive(): self._submit("stop") self._event_thread.join(timeout=35.0) if self._event_thread.is_alive(): self.manager.last_warning = "event consumer did not stop within 35s" raise RuntimeError(self.manager.last_warning) else: self._event_thread = None self._event_stop.set() if self._evidence is not None: self._evidence.stop(timeout=3.0) self.manager.stop() # Point flow.json v2 fields (already present from create_flow). try: flow = art.load_flow(self.flow_dir) v2 = dict(flow.get("v2") or {}) v2["action_count"] = len(self.manager.action_flow.actions) v2["state_count"] = len(self.manager.action_flow.states) v2["observation_count"] = len( self.manager.snapshots.observation_ids() ) v2["element_count"] = len(self.manager.catalog.catalog.elements) v2["raw_event_journal"] = "journal/raw_events.jsonl" v2["pages"] = "pages.json" v2["diagnostics_dir"] = "diagnostics/" v2["validation"] = "diagnostics/validation.json" v2["replay_report"] = "diagnostics/replay_report.json" flow["v2"] = v2 art.save_flow(self.flow_dir, flow) except Exception: pass self._write_diagnostics() self._refresh_stats() if recorder_stop_error is not None: self.manager.last_warning = str(recorder_stop_error) raise RuntimeError(str(recorder_stop_error)) from recorder_stop_error def checkpoint(self) -> Any: return self.manager.record_checkpoint() def heartbeat(self) -> None: self._submit("heartbeat") def stats(self) -> dict[str, Any]: with self._stats_lock: return dict(self._stats_cache) def _refresh_stats(self) -> None: st = self.manager.stats() st["surface_recorder"] = ( self._recorder.status if self._recorder else "off" ) st["queued_events"] = self._event_queue.qsize() st["dropped_events"] = self._dropped_events if self._recorder is not None and hasattr(self._recorder, "health"): try: st["input_backend"] = self._recorder.health() except Exception as exc: st["input_backend"] = {"last_error": str(exc)} elif self._recorder_health: st["input_backend"] = dict(self._recorder_health) if self._evidence is not None: st["evidence_capture_count"] = self._evidence.capture_count st["evidence_dropped"] = self._evidence.dropped st["evidence_queued"] = self._evidence._queue.qsize() st["transient_evidence_queued"] = self._evidence._urgent_queue.qsize() st["evidence_failures"] = self._evidence.failure_count with self._stats_lock: self._stats_cache = st def _write_diagnostics(self) -> None: pages: dict[str, dict[str, Any]] = {} lags: list[int] = [] for action in self.manager._actions_by_id.values(): raw = dict(action.raw_event or {}) meta = dict(raw.get("meta") or {}) page_id = str(meta.get("page_id") or "") if page_id: page = pages.setdefault(page_id, {"page_id": page_id, "urls": [], "document_ids": []}) url = str(raw.get("url") or (action.input or {}).get("url") or "") document_id = str(meta.get("document_id") or "") if url and url not in page["urls"]: page["urls"].append(url) if document_id and document_id not in page["document_ids"]: page["document_ids"].append(document_id) lag = (action.evidence or {}).get("evidence_lag_ms") if isinstance(lag, int): lags.append(lag) diagnostics = self.flow_dir / "diagnostics" diagnostics.mkdir(parents=True, exist_ok=True) try: art.write_json(self.flow_dir / "pages.json", { "schema": "openclaw.ui_inspector.pages.v1", "pages": list(pages.values()), }) recorder_health = dict(self._recorder_health) if self._recorder is not None and hasattr(self._recorder, "health"): try: recorder_health = self._recorder.health() except Exception as exc: recorder_health = {"last_error": str(exc)} art.write_json(diagnostics / "metrics.json", { "schema": "openclaw.ui_inspector.metrics.v1", "raw_event_count": self.journal._sequence, "action_count": len(self.manager.action_flow.actions), "state_count": len(self.manager.action_flow.states), "observation_count": len(self.manager.snapshots.observation_ids()), "element_count": len(self.manager.catalog.catalog.elements), "event_queue_dropped": self._dropped_events, "evidence_queue_dropped": self._evidence.dropped if self._evidence else 0, "evidence_capture_count": self._evidence.capture_count if self._evidence else 0, "evidence_failure_count": self._evidence.failure_count if self._evidence else 0, "evidence_lag_ms_max": max(lags) if lags else None, "evidence_lag_ms_average": round(sum(lags) / len(lags), 2) if lags else None, "input_backend": recorder_health, }) from diagnostics import validate_capture, write_replay_preflight art.write_json(diagnostics / "validation.json", validate_capture(self.flow_dir)) write_replay_preflight(self.flow_dir) except Exception as exc: self.manager.last_warning = f"diagnostics write failed: {exc}"