chore: archive tools/ui_inspector snapshot before migration removal.

Keep the last in-template copy on main so history can restore it; product continues in jiangchang-ui-recorder.

Co-authored-by: Cursor <cursoragent@cursor.com>

tools/ui_inspector/flow_session.py

@@ -0,0 +1,408 @@
+"""Flow session glue: v2 ActionRecordingManager + surface recorders + v1 dual-write."""
+
+from __future__ import annotations
+
+import queue
+import threading
+import uuid
+from pathlib import Path
+from typing import Any, Callable, Optional
+
+import artifacts as art
+from core.pipeline.event_journal import EventJournal
+from evidence.coordinator import EvidenceCoordinator
+from recorder.base import RawUserEvent
+from recorder.manager import ActionRecordingManager
+
+
+class FlowSession:
+    def __init__(
+        self,
+        flow_dir: Path,
+        *,
+        flow_name: str,
+        surface: str,
+        recording_mode: str = "auto",
+        dump_fn: Optional[Callable[[], Any]] = None,
+        write_v1_step: Optional[Callable[..., Any]] = None,
+        exclude_hwnds: Optional[Callable[[], set[int]]] = None,
+        on_action_update: Optional[Callable[[Any, str], None]] = None,
+        stability_timeout_sec: float = 3.0,
+    ) -> None:
+        self.flow_dir = Path(flow_dir)
+        self.surface = surface
+        self.recording_mode = recording_mode
+        self._dump_fn = dump_fn
+        self.exclude_hwnds = exclude_hwnds or (lambda: set())
+        self.manager = ActionRecordingManager(
+            self.flow_dir,
+            flow_name=flow_name,
+            flow_id=self.flow_dir.name,
+            surface=surface,
+            recording_mode=recording_mode,
+            dump_fn=dump_fn,
+            write_v1_step=write_v1_step,
+            stability_timeout_sec=stability_timeout_sec,
+            defer_action_evidence=(
+                surface in {"browser", "windows"}
+                and recording_mode != "legacy_interval"
+            ),
+            on_action_update=on_action_update,
+        )
+        self.journal = EventJournal(self.flow_dir)
+        self._evidence = (
+            EvidenceCoordinator(self.manager, dump_fn, on_dump=self._on_evidence_dump)
+            if surface in {"browser", "windows"}
+            and dump_fn is not None
+            and recording_mode != "legacy_interval"
+            else None
+        )
+        if self._evidence is not None:
+            self.manager.on_action = self._evidence.schedule
+        self._recorder = None
+        self._recorder_health: dict[str, Any] = {}
+        self._event_queue: queue.Queue[tuple[str, Any]] = queue.Queue(maxsize=512)
+        self._event_thread: Optional[threading.Thread] = None
+        self._event_stop = threading.Event()
+        self._stats_lock = threading.Lock()
+        self._stats_cache: dict[str, Any] = {
+            "recorder_status": "idle",
+            "surface_recorder": "off",
+            "state_count": 0,
+            "observation_count": 0,
+            "action_count": 0,
+            "element_count": 0,
+            "queued_events": 0,
+            "dropped_events": 0,
+        }
+        self._dropped_events = 0
+
+    def start(self, *, entry_dump: Optional[Any] = None) -> None:
+        if self.recording_mode == "legacy_interval":
+            self.manager.start(entry_dump=entry_dump)
+            self._refresh_stats()
+            return
+        # Start observation before entry persistence. Events are journaled and
+        # buffered until manager.start() completes, eliminating startup loss.
+        self._start_surface_recorder()
+        if self.surface == "windows" and entry_dump is not None:
+            self._on_evidence_dump(entry_dump)
+        if self._evidence is not None:
+            self._evidence.start()
+        try:
+            self.manager.start(entry_dump=entry_dump)
+        except Exception:
+            if self._recorder:
+                self._recorder.stop()
+                self._recorder = None
+            if self._evidence is not None:
+                self._evidence.stop(timeout=3.0)
+            raise
+        self._start_event_consumer()
+        if self.surface == "windows" and self._evidence is not None:
+            # Warm/enrich UIA immediately, but on the evidence worker. Raw
+            # recording and Action reduction remain fully independent.
+            self._evidence.prime()
+        self._refresh_stats()
+
+    def _submit(self, kind: str, payload: Any = None) -> bool:
+        try:
+            if kind == "stop":
+                self._event_queue.put((kind, payload), timeout=2.0)
+            else:
+                self._event_queue.put_nowait((kind, payload))
+            return True
+        except queue.Full:
+            self._dropped_events += 1
+            self.manager.last_warning = (
+                f"recorder queue full; dropped={self._dropped_events}"
+            )
+            self._refresh_stats()
+            return False
+
+    def _start_event_consumer(self) -> None:
+        if self._event_thread and self._event_thread.is_alive():
+            return
+        self._event_stop.clear()
+
+        def consume() -> None:
+            while not self._event_stop.is_set():
+                try:
+                    kind, payload = self._event_queue.get(timeout=0.2)
+                except queue.Empty:
+                    continue
+                try:
+                    if kind == "stop":
+                        break
+                    if kind == "event":
+                        self.manager.handle_raw_event(payload)
+                    elif kind == "heartbeat":
+                        self.manager.heartbeat()
+                    elif kind == "pause":
+                        self.manager.pause()
+                    elif kind == "resume":
+                        self.manager.resume()
+                except Exception as exc:
+                    self.manager.last_warning = str(exc)
+                finally:
+                    self._event_queue.task_done()
+                    self._refresh_stats()
+            self._event_stop.set()
+
+        self._event_thread = threading.Thread(
+            target=consume,
+            name="ui-inspector-action-consumer",
+            daemon=True,
+        )
+        self._event_thread.start()
+
+    def _start_surface_recorder(self) -> None:
+        def on_event(ev: RawUserEvent) -> None:
+            # Preserve the observed order and bound memory/thread growth.
+            if self.surface == "windows" and not (ev.meta or {}).get("event_id"):
+                ev.meta["event_id"] = f"windows:{uuid.uuid4().hex}"
+            self.journal.append(ev, channel="semantic")
+            self._submit("event", ev)
+
+        def on_raw_event(payload: dict[str, Any]) -> None:
+            self.journal.append(payload, channel="browser_dom")
+
+        if self.surface == "browser":
+            from surfaces.browser import BrowserRecorder
+
+            self._recorder = BrowserRecorder(
+                on_event=on_event, on_raw_event=on_raw_event
+            )
+        else:
+            from surfaces.windows import WindowsRecorder
+
+            self._recorder = WindowsRecorder(
+                on_event=on_event,
+                exclude_hwnds=self.exclude_hwnds,
+            )
+            self._seed_windows_element_atlas()
+        self._recorder.start()
+
+    def _on_evidence_dump(self, dump: Any) -> None:
+        recorder = self._recorder
+        if self.surface != "windows" or recorder is None or not hasattr(recorder, "learn_elements"):
+            return
+        try:
+            recorder.learn_elements(
+                list(getattr(dump, "locatable", None) or []),
+                window=dict(getattr(dump, "window", None) or {}),
+            )
+        except Exception:
+            pass
+
+    def _seed_windows_element_atlas(self) -> None:
+        """Reuse recent captures as the event-time element repository.
+
+        Values are discarded by ``learn_elements``; only stable names and
+        geometry are retained. This lets a new recording identify custom
+        editors before their slow accessibility provider completes a new dump.
+        """
+        recorder = self._recorder
+        if recorder is None or not hasattr(recorder, "learn_elements"):
+            return
+        try:
+            siblings = sorted(
+                (
+                    path for path in self.flow_dir.parent.iterdir()
+                    if path.is_dir() and path != self.flow_dir
+                ),
+                key=lambda path: path.stat().st_mtime,
+                reverse=True,
+            )[:8]
+        except Exception:
+            return
+        learned_classes: set[str] = set()
+        for sibling in siblings:
+            observations = sibling / "observations"
+            if not observations.is_dir():
+                continue
+            try:
+                metas = sorted(
+                    observations.glob("*/meta.json"),
+                    key=lambda path: path.stat().st_mtime,
+                    reverse=True,
+                )
+            except Exception:
+                continue
+            for meta_path in metas:
+                try:
+                    import json
+
+                    meta = json.loads(meta_path.read_text(encoding="utf-8"))
+                    if str(meta.get("surface") or "") != "windows":
+                        continue
+                    selection = dict(meta.get("selection") or {})
+                    class_name = str(
+                        selection.get("chosen_class")
+                        or selection.get("event_class")
+                        or ""
+                    )
+                    if not class_name or class_name in learned_classes:
+                        continue
+                    locatable_path = meta_path.parent / "locatable.json"
+                    rows = json.loads(locatable_path.read_text(encoding="utf-8"))
+                    recorder.learn_elements(
+                        rows,
+                        window={
+                            "class_name": class_name,
+                            "hwnd": int(selection.get("chosen_hwnd") or 0),
+                            "pid": int(selection.get("chosen_pid") or 0),
+                            "title": str(selection.get("event_title") or ""),
+                        },
+                    )
+                    learned_classes.add(class_name)
+                except Exception:
+                    continue
+            if "WeWorkWindow" in learned_classes and "SearchResultWindow2" in learned_classes:
+                break
+
+    def pause(self) -> None:
+        if self._recorder:
+            self._recorder.pause()
+        self._submit("pause")
+
+    def resume(self) -> None:
+        if self._recorder:
+            self._recorder.resume()
+        self._submit("resume")
+
+    def stop(self) -> None:
+        recorder_stop_error: Optional[BaseException] = None
+        if self._recorder:
+            try:
+                self._recorder.stop()
+            except Exception as exc:
+                recorder_stop_error = exc
+            if hasattr(self._recorder, "health"):
+                try:
+                    self._recorder_health = self._recorder.health()
+                except Exception as exc:
+                    self._recorder_health = {"last_error": str(exc)}
+            self._recorder = None
+        if self._event_thread and self._event_thread.is_alive():
+            self._submit("stop")
+            self._event_thread.join(timeout=35.0)
+            if self._event_thread.is_alive():
+                self.manager.last_warning = "event consumer did not stop within 35s"
+                raise RuntimeError(self.manager.last_warning)
+            else:
+                self._event_thread = None
+        self._event_stop.set()
+        if self._evidence is not None:
+            self._evidence.stop(timeout=3.0)
+        self.manager.stop()
+        # Point flow.json v2 fields (already present from create_flow).
+        try:
+            flow = art.load_flow(self.flow_dir)
+            v2 = dict(flow.get("v2") or {})
+            v2["action_count"] = len(self.manager.action_flow.actions)
+            v2["state_count"] = len(self.manager.action_flow.states)
+            v2["observation_count"] = len(
+                self.manager.snapshots.observation_ids()
+            )
+            v2["element_count"] = len(self.manager.catalog.catalog.elements)
+            v2["raw_event_journal"] = "journal/raw_events.jsonl"
+            v2["pages"] = "pages.json"
+            v2["diagnostics_dir"] = "diagnostics/"
+            v2["validation"] = "diagnostics/validation.json"
+            v2["replay_report"] = "diagnostics/replay_report.json"
+            flow["v2"] = v2
+            art.save_flow(self.flow_dir, flow)
+        except Exception:
+            pass
+        self._write_diagnostics()
+        self._refresh_stats()
+        if recorder_stop_error is not None:
+            self.manager.last_warning = str(recorder_stop_error)
+            raise RuntimeError(str(recorder_stop_error)) from recorder_stop_error
+
+    def checkpoint(self) -> Any:
+        return self.manager.record_checkpoint()
+
+    def heartbeat(self) -> None:
+        self._submit("heartbeat")
+
+    def stats(self) -> dict[str, Any]:
+        with self._stats_lock:
+            return dict(self._stats_cache)
+
+    def _refresh_stats(self) -> None:
+        st = self.manager.stats()
+        st["surface_recorder"] = (
+            self._recorder.status if self._recorder else "off"
+        )
+        st["queued_events"] = self._event_queue.qsize()
+        st["dropped_events"] = self._dropped_events
+        if self._recorder is not None and hasattr(self._recorder, "health"):
+            try:
+                st["input_backend"] = self._recorder.health()
+            except Exception as exc:
+                st["input_backend"] = {"last_error": str(exc)}
+        elif self._recorder_health:
+            st["input_backend"] = dict(self._recorder_health)
+        if self._evidence is not None:
+            st["evidence_capture_count"] = self._evidence.capture_count
+            st["evidence_dropped"] = self._evidence.dropped
+            st["evidence_queued"] = self._evidence._queue.qsize()
+            st["transient_evidence_queued"] = self._evidence._urgent_queue.qsize()
+            st["evidence_failures"] = self._evidence.failure_count
+        with self._stats_lock:
+            self._stats_cache = st
+
+    def _write_diagnostics(self) -> None:
+        pages: dict[str, dict[str, Any]] = {}
+        lags: list[int] = []
+        for action in self.manager._actions_by_id.values():
+            raw = dict(action.raw_event or {})
+            meta = dict(raw.get("meta") or {})
+            page_id = str(meta.get("page_id") or "")
+            if page_id:
+                page = pages.setdefault(page_id, {"page_id": page_id, "urls": [], "document_ids": []})
+                url = str(raw.get("url") or (action.input or {}).get("url") or "")
+                document_id = str(meta.get("document_id") or "")
+                if url and url not in page["urls"]:
+                    page["urls"].append(url)
+                if document_id and document_id not in page["document_ids"]:
+                    page["document_ids"].append(document_id)
+            lag = (action.evidence or {}).get("evidence_lag_ms")
+            if isinstance(lag, int):
+                lags.append(lag)
+        diagnostics = self.flow_dir / "diagnostics"
+        diagnostics.mkdir(parents=True, exist_ok=True)
+        try:
+            art.write_json(self.flow_dir / "pages.json", {
+                "schema": "openclaw.ui_inspector.pages.v1",
+                "pages": list(pages.values()),
+            })
+            recorder_health = dict(self._recorder_health)
+            if self._recorder is not None and hasattr(self._recorder, "health"):
+                try:
+                    recorder_health = self._recorder.health()
+                except Exception as exc:
+                    recorder_health = {"last_error": str(exc)}
+            art.write_json(diagnostics / "metrics.json", {
+                "schema": "openclaw.ui_inspector.metrics.v1",
+                "raw_event_count": self.journal._sequence,
+                "action_count": len(self.manager.action_flow.actions),
+                "state_count": len(self.manager.action_flow.states),
+                "observation_count": len(self.manager.snapshots.observation_ids()),
+                "element_count": len(self.manager.catalog.catalog.elements),
+                "event_queue_dropped": self._dropped_events,
+                "evidence_queue_dropped": self._evidence.dropped if self._evidence else 0,
+                "evidence_capture_count": self._evidence.capture_count if self._evidence else 0,
+                "evidence_failure_count": self._evidence.failure_count if self._evidence else 0,
+                "evidence_lag_ms_max": max(lags) if lags else None,
+                "evidence_lag_ms_average": round(sum(lags) / len(lags), 2) if lags else None,
+                "input_backend": recorder_health,
+            })
+            from diagnostics import validate_capture, write_replay_preflight
+
+            art.write_json(diagnostics / "validation.json", validate_capture(self.flow_dir))
+            write_replay_preflight(self.flow_dir)
+        except Exception as exc:
+            self.manager.last_warning = f"diagnostics write failed: {exc}"