chore: archive tools/ui_inspector snapshot before migration removal.

Keep the last in-template copy on main so history can restore it; product continues in jiangchang-ui-recorder.

Co-authored-by: Cursor <cursoragent@cursor.com>

tools/ui_inspector/models/__init__.py

@@ -0,0 +1,22 @@
+"""UI Inspector v2 data models (Action / State / Element / Flow)."""
+
+from .action import ACTION_TYPES, ActionFlow, ActionRecord
+from .element import ElementRef
+from .locator import LocatorCandidate
+from .observation import UIObservation
+from .serialize import from_dict, to_dict, write_model_json
+from .state import UIState, compute_state_fingerprint
+
+__all__ = [
+    "ACTION_TYPES",
+    "ActionFlow",
+    "ActionRecord",
+    "ElementRef",
+    "LocatorCandidate",
+    "UIState",
+    "UIObservation",
+    "compute_state_fingerprint",
+    "from_dict",
+    "to_dict",
+    "write_model_json",
+]

tools/ui_inspector/models/action.py

@@ -0,0 +1,119 @@
+"""ActionRecord + ActionFlow models."""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Optional
+
+SCHEMA_ACTION = "openclaw.ui_inspector.action.v1"
+SCHEMA_ACTION_FLOW = "openclaw.ui_inspector.action_flow.v1"
+
+# Full vocabulary — capability flags live elsewhere for unimplemented types.
+ACTION_TYPES: tuple[str, ...] = (
+    "click",
+    "double_click",
+    "right_click",
+    "fill",
+    "type",
+    "clear",
+    "press_key",
+    "hotkey",
+    "select",
+    "toggle",
+    "hover",
+    "scroll",
+    "drag_drop",
+    "focus",
+    "window_switch",
+    "navigation",
+    "checkpoint",
+    "wait",
+    "verify",
+    "unknown",
+)
+
+# Capability status for this Inspector release.
+ACTION_CAPABILITY: dict[str, str] = {
+    "click": "implemented",
+    "double_click": "schema_only",
+    "right_click": "schema_only",
+    "fill": "implemented",
+    "type": "implemented",  # aggregated into fill/type
+    "clear": "schema_only",
+    "press_key": "implemented",
+    "hotkey": "schema_only",
+    "select": "schema_only",
+    "toggle": "schema_only",
+    "hover": "schema_only",
+    "scroll": "schema_only",
+    "drag_drop": "schema_only",
+    "focus": "schema_only",
+    "window_switch": "schema_only",
+    "navigation": "partial",
+    "checkpoint": "implemented",
+    "wait": "schema_only",
+    "verify": "schema_only",
+    "unknown": "fallback",
+}
+
+
+def _now_iso() -> str:
+    return datetime.now().astimezone().isoformat()
+
+
+def _new_id(prefix: str = "act") -> str:
+    return f"{prefix}_{uuid.uuid4().hex[:12]}"
+
+
+@dataclass
+class ActionRecord:
+    schema: str = SCHEMA_ACTION
+    id: str = field(default_factory=lambda: _new_id("act"))
+    sequence: int = 0
+    action_type: str = "unknown"
+    started_at: str = field(default_factory=_now_iso)
+    ended_at: Optional[str] = None
+    surface: str = "windows"
+    target_element_id: Optional[str] = None
+    target_snapshot: dict[str, Any] = field(default_factory=dict)
+    input: dict[str, Any] = field(default_factory=dict)
+    sensitive: bool = False
+    before_state_id: Optional[str] = None
+    after_state_id: Optional[str] = None
+    before_observation_id: Optional[str] = None
+    after_observation_id: Optional[str] = None
+    observed_changes: list[dict[str, Any]] = field(default_factory=list)
+    proposed_postconditions: list[dict[str, Any]] = field(default_factory=list)
+    status: str = "ok"  # ok | partial | failed | skipped
+    confidence: float = 0.0
+    evidence: dict[str, Any] = field(default_factory=dict)
+    raw_event: dict[str, Any] = field(default_factory=dict)
+    raw_event_ids: list[str] = field(default_factory=list)
+    capability: str = ""
+
+    def __post_init__(self) -> None:
+        if self.action_type not in ACTION_TYPES:
+            self.action_type = "unknown"
+        if not self.capability:
+            self.capability = ACTION_CAPABILITY.get(self.action_type, "schema_only")
+
+
+@dataclass
+class ActionFlow:
+    schema: str = SCHEMA_ACTION_FLOW
+    flow_id: str = ""
+    flow_name: str = ""
+    created_at: str = field(default_factory=_now_iso)
+    ended_at: Optional[str] = None
+    states: list[str] = field(default_factory=list)  # state ids in order seen
+    observations: list[str] = field(default_factory=list)  # every concrete capture
+    actions: list[str] = field(default_factory=list)  # action ids in sequence
+    elements: list[str] = field(default_factory=list)  # element ids
+    entry_state_id: Optional[str] = None
+    final_state_id: Optional[str] = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    recording_mode: str = "auto"  # auto | guided | checkpoint | legacy_interval
+    surface: str = ""
+    status: str = "open"

tools/ui_inspector/models/element.py

@@ -0,0 +1,126 @@
+"""ElementRef model for Element Catalog."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Optional
+
+from .locator import LocatorCandidate
+
+SCHEMA_ELEMENT = "openclaw.ui_inspector.element.v1"
+SCHEMA_CATALOG = "openclaw.ui_inspector.element_catalog.v1"
+
+
+def _now_iso() -> str:
+    return datetime.now().astimezone().isoformat()
+
+
+def _new_id(prefix: str = "el") -> str:
+    return f"{prefix}_{uuid.uuid4().hex[:12]}"
+
+
+def element_fingerprint(
+    *,
+    surface: str,
+    role: str = "",
+    name: str = "",
+    automation_id: str = "",
+    class_name: str = "",
+    value: str = "",
+    window_scope: Optional[dict[str, Any]] = None,
+    dom_fingerprint: str = "",
+    page_scope: str = "",
+    contenteditable: bool = False,
+) -> str:
+    payload = {
+        "surface": surface or "",
+        "role": role or "",
+        "name": "" if contenteditable else (name or ""),
+        "automation_id": automation_id or "",
+        "class_name": class_name or "",
+        # Value is identity only when short/stable-looking (not long free text).
+        "value": "" if contenteditable else ((value or "") if len(value or "") <= 64 else ""),
+        "dom_fingerprint": dom_fingerprint or "",
+        "page_scope": page_scope or "",
+        "window_scope": {
+            "class_name": (window_scope or {}).get("class_name") or "",
+            "title": (window_scope or {}).get("title") or "",
+        },
+    }
+    raw = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(raw.encode("utf-8")).hexdigest()
+
+
+@dataclass
+class ElementRef:
+    schema: str = SCHEMA_ELEMENT
+    id: str = field(default_factory=lambda: _new_id("el"))
+    logical_name: str = ""
+    surface: str = "windows"
+    backend: str = ""
+    app: dict[str, Any] = field(default_factory=dict)
+    window_scope: dict[str, Any] = field(default_factory=dict)
+    element_scope: dict[str, Any] = field(default_factory=dict)
+    role: str = ""
+    control_type: str = ""
+    name: str = ""
+    value: str = ""
+    automation_id: str = ""
+    class_name: str = ""
+    framework_id: str = ""
+    states: dict[str, Any] = field(default_factory=dict)
+    bounds: dict[str, Any] = field(default_factory=dict)
+    actions: list[str] = field(default_factory=list)
+    parent_chain: list[dict[str, Any]] = field(default_factory=list)
+    raw: dict[str, Any] = field(default_factory=dict)
+    fingerprint: str = ""
+    sensitive: bool = False
+    locator_candidates: list[LocatorCandidate] = field(default_factory=list)
+    primary_locator: Optional[LocatorCandidate] = None
+    fallback_locators: list[LocatorCandidate] = field(default_factory=list)
+    expected_count: int = 1
+    verified_count: Optional[int] = None
+    verification_level: str = "none"
+    stability_score: float = 0.0
+    verified_at: Optional[str] = None
+    first_seen: str = field(default_factory=_now_iso)
+    last_seen: str = field(default_factory=_now_iso)
+    observation_count: int = 1
+    locator_history: list[dict[str, Any]] = field(default_factory=list)
+    parameterized: dict[str, str] = field(default_factory=dict)
+
+    def ensure_fingerprint(self) -> str:
+        if self.fingerprint:
+            return self.fingerprint
+        self.fingerprint = element_fingerprint(
+            surface=self.surface,
+            role=self.role or self.control_type,
+            name=self.name,
+            automation_id=self.automation_id,
+            class_name=self.class_name,
+            value=self.value,
+            window_scope=self.window_scope,
+            dom_fingerprint=str(self.raw.get("fingerprint") or self.raw.get("css") or ""),
+            page_scope=str(self.element_scope.get("page_url") or ""),
+            contenteditable=bool(self.raw.get("contenteditable")),
+        )
+        return self.fingerprint
+
+    def touch(self) -> None:
+        self.last_seen = _now_iso()
+        self.observation_count = int(self.observation_count or 0) + 1
+
+
+@dataclass
+class ElementCatalog:
+    schema: str = SCHEMA_CATALOG
+    flow_id: str = ""
+    updated_at: str = field(default_factory=_now_iso)
+    elements: list[ElementRef] = field(default_factory=list)
+
+    def by_logical_name(self) -> dict[str, ElementRef]:
+        return {e.logical_name: e for e in self.elements if e.logical_name}

tools/ui_inspector/models/flow.py

@@ -0,0 +1,11 @@
+"""Flow-level helpers (re-export ActionFlow for models.flow import path)."""
+
+from .action import ActionFlow, ActionRecord
+from .element import ElementCatalog, ElementRef
+from .state import UIState
+from .observation import UIObservation
+
+__all__ = [
+    "ActionFlow", "ActionRecord", "ElementCatalog", "ElementRef", "UIState",
+    "UIObservation",
+]

tools/ui_inspector/models/locator.py

@@ -0,0 +1,40 @@
+"""Structured locator candidate model."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+
+SCHEMA_LOCATOR = "openclaw.ui_inspector.locator_candidate.v1"
+
+
+@dataclass
+class LocatorCandidate:
+    schema: str = SCHEMA_LOCATOR
+    strategy: str = ""
+    scope: dict[str, Any] = field(default_factory=dict)
+    properties: dict[str, Any] = field(default_factory=dict)
+    expression: str = ""
+    expected_count: int = 1
+    actual_count: Optional[int] = None
+    unique: Optional[bool] = None
+    stability_score: float = 0.0
+    uses_index: bool = False
+    uses_coordinates: bool = False
+    validation: str = "none"  # snapshot_only | live_tree | cross_state | cross_session | none
+    reasons: list[str] = field(default_factory=list)
+    ambiguous: bool = False
+    production_eligible: bool = False
+
+    def mark_counts(self, actual: int, *, expected: int = 1) -> None:
+        self.expected_count = int(expected)
+        self.actual_count = int(actual)
+        self.unique = self.actual_count == self.expected_count
+        self.ambiguous = not bool(self.unique)
+        # Coordinates / index never become production primary automatically.
+        self.production_eligible = bool(
+            self.unique
+            and not self.uses_coordinates
+            and not (self.uses_index and self.stability_score < 0.7)
+        )

tools/ui_inspector/models/observation.py

@@ -0,0 +1,47 @@
+"""Per-capture evidence model.
+
+UIState represents a deduplicated semantic state.  UIObservation represents
+one concrete capture and is therefore never deduplicated.
+"""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Optional
+
+SCHEMA_OBSERVATION = "openclaw.ui_inspector.observation.v1"
+
+
+def _now_iso() -> str:
+    return datetime.now().astimezone().isoformat()
+
+
+def _new_id() -> str:
+    return f"obs_{uuid.uuid4().hex[:12]}"
+
+
+@dataclass
+class UIObservation:
+    schema: str = SCHEMA_OBSERVATION
+    id: str = field(default_factory=_new_id)
+    captured_at: str = field(default_factory=_now_iso)
+    state_id: str = ""
+    previous_observation_id: Optional[str] = None
+    change_reason: str = "capture"
+    surface: str = "windows"
+    driver: str = ""
+    page_id: str = ""
+    document_id: str = ""
+    frame_id: str = ""
+    url: str = ""
+    page_title: str = ""
+    tree_path: str = ""
+    index_path: str = ""
+    locatable_path: str = ""
+    screenshot_path: str = ""
+    stats: dict[str, Any] = field(default_factory=dict)
+    selection: dict[str, Any] = field(default_factory=dict)
+    may_contain_sensitive_data: bool = False
+

tools/ui_inspector/models/serialize.py

@@ -0,0 +1,64 @@
+"""Stable JSON serialization helpers for v2 models."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, is_dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import Any, TypeVar
+
+T = TypeVar("T")
+
+
+def _normalize(obj: Any) -> Any:
+    if obj is None or isinstance(obj, (str, int, float, bool)):
+        return obj
+    if isinstance(obj, datetime):
+        return obj.isoformat()
+    if isinstance(obj, Path):
+        return str(obj).replace("\\", "/")
+    if is_dataclass(obj) and not isinstance(obj, type):
+        return {k: _normalize(v) for k, v in asdict(obj).items()}
+    if isinstance(obj, dict):
+        return {str(k): _normalize(v) for k, v in obj.items()}
+    if isinstance(obj, (list, tuple)):
+        return [_normalize(x) for x in obj]
+    return str(obj)
+
+
+def to_dict(obj: Any) -> dict[str, Any]:
+    data = _normalize(obj)
+    if not isinstance(data, dict):
+        raise TypeError(f"expected dataclass/dict, got {type(obj)!r}")
+    return data
+
+
+def from_dict(cls: type[T], data: dict[str, Any]) -> T:
+    """Best-effort dataclass reconstruction (ignores unknown keys)."""
+    if not is_dataclass(cls):
+        raise TypeError(f"{cls!r} is not a dataclass")
+    import dataclasses
+
+    fields = {f.name for f in dataclasses.fields(cls)}
+    kwargs = {k: v for k, v in (data or {}).items() if k in fields}
+    return cls(**kwargs)  # type: ignore[call-arg]
+
+
+def dumps_json(obj: Any, *, indent: int = 2) -> str:
+    return json.dumps(
+        to_dict(obj) if is_dataclass(obj) and not isinstance(obj, type) else _normalize(obj),
+        ensure_ascii=False,
+        indent=indent,
+        sort_keys=False,
+    ) + "\n"
+
+
+def write_model_json(path: Path, obj: Any) -> None:
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(dumps_json(obj), encoding="utf-8")
+
+
+def read_json(path: Path) -> Any:
+    return json.loads(Path(path).read_text(encoding="utf-8"))

tools/ui_inspector/models/state.py

@@ -0,0 +1,175 @@
+"""UIState model + stable business fingerprint."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Optional
+
+SCHEMA_STATE = "openclaw.ui_inspector.state.v1"
+
+# Keys ignored when hashing business identity.
+_FINGERPRINT_EXCLUDE_TOP = {
+    "captured_at",
+    "id",
+    "previous_state_id",
+    "tree_path",
+    "index_path",
+    "locatable_path",
+    "screenshot_path",
+    "duplicate_of",
+    "observation_count",
+    "change_reason",
+}
+
+_SESSION_KEYS = {
+    "pid",
+    "hwnd",
+    "process_id",
+    "thread_id",
+    "elapsed_ms",
+    "dump_ms",
+    "warmup_ms",
+    "timings",
+    "screenshot",
+    "screen_path",
+}
+
+
+def _now_iso() -> str:
+    return datetime.now().astimezone().isoformat()
+
+
+def _new_id(prefix: str = "st") -> str:
+    return f"{prefix}_{uuid.uuid4().hex[:12]}"
+
+
+def _strip_session(obj: Any) -> Any:
+    if isinstance(obj, dict):
+        out: dict[str, Any] = {}
+        for k, v in obj.items():
+            lk = str(k).lower()
+            if lk in _SESSION_KEYS or lk.endswith("_ms") or lk.endswith("_path"):
+                continue
+            if lk in {"left", "top", "right", "bottom", "x", "y", "width", "height"}:
+                # Absolute coordinates are not primary identity.
+                continue
+            if lk == "rect" or lk == "bounds" or lk == "bounding_rectangle":
+                continue
+            out[str(k)] = _strip_session(v)
+        return out
+    if isinstance(obj, list):
+        return [_strip_session(x) for x in obj]
+    return obj
+
+
+def _locatable_signature(rows: list[dict[str, Any]], *, limit: int = 120) -> list[dict[str, Any]]:
+    """Compact visible-business signature for fingerprinting."""
+    sig: list[dict[str, Any]] = []
+    for r in rows[:limit]:
+        if not isinstance(r, dict):
+            continue
+        entry = {
+            "control_type": r.get("control_type") or r.get("role") or "",
+            "name": r.get("name") or "",
+            "value": r.get("value") or r.get("text") or "",
+            "automation_id": r.get("automation_id") or r.get("test_id") or "",
+            "class_name": r.get("class_name") or "",
+        }
+        # Drop empty noise rows
+        if not any(str(v).strip() for v in entry.values()):
+            continue
+        sig.append(entry)
+    # Stable order
+    sig.sort(
+        key=lambda e: (
+            str(e.get("control_type") or ""),
+            str(e.get("automation_id") or ""),
+            str(e.get("name") or ""),
+            str(e.get("value") or ""),
+        )
+    )
+    return sig
+
+
+def compute_state_fingerprint(
+    *,
+    surface: str,
+    driver: str = "",
+    url: str = "",
+    page_title: str = "",
+    window: Optional[dict[str, Any]] = None,
+    locatable: Optional[list[dict[str, Any]]] = None,
+    app: Optional[dict[str, Any]] = None,
+    extra_business: Optional[dict[str, Any]] = None,
+) -> str:
+    """
+    SHA-256 of stable business identity.
+
+    Excludes timestamps, pid/hwnd, screenshot paths, absolute coords, timings.
+    """
+    win = dict(window or {})
+    app_d = dict(app or {})
+    payload = {
+        "surface": surface or "",
+        "driver": driver or "",
+        "url": url or win.get("url") or app_d.get("url") or "",
+        "page_title": page_title or "",
+        "window": {
+            "title": win.get("title") or app_d.get("title") or "",
+            "class_name": win.get("class_name") or app_d.get("class_name") or "",
+            "process_name": win.get("process_name") or app_d.get("process_name") or "",
+        },
+        "locatable": _locatable_signature(list(locatable or [])),
+        "extra": _strip_session(extra_business or {}),
+    }
+    raw = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(raw.encode("utf-8")).hexdigest()
+
+
+@dataclass
+class UIState:
+    schema: str = SCHEMA_STATE
+    id: str = field(default_factory=lambda: _new_id("st"))
+    surface: str = "windows"
+    driver: str = ""
+    captured_at: str = field(default_factory=_now_iso)
+    fingerprint: str = ""
+    previous_state_id: Optional[str] = None
+    change_reason: str = ""
+    app: dict[str, Any] = field(default_factory=dict)
+    window: dict[str, Any] = field(default_factory=dict)
+    url: str = ""
+    page_title: str = ""
+    tree_path: str = ""
+    index_path: str = ""
+    locatable_path: str = ""
+    screenshot_path: str = ""
+    stats: dict[str, Any] = field(default_factory=dict)
+    selection: dict[str, Any] = field(default_factory=dict)
+    duplicate_of: Optional[str] = None
+    observation_count: int = 1
+    may_contain_sensitive_data: bool = False
+    # In-memory locatable used for fingerprint/diff (not always persisted as field).
+    locatable: list[dict[str, Any]] = field(default_factory=list, repr=False)
+
+    def ensure_fingerprint(self) -> str:
+        if self.fingerprint:
+            return self.fingerprint
+        self.fingerprint = compute_state_fingerprint(
+            surface=self.surface,
+            driver=self.driver,
+            url=self.url,
+            page_title=self.page_title,
+            window=self.window,
+            locatable=self.locatable,
+            app=self.app,
+        )
+        return self.fingerprint
+
+    def bump_observation(self) -> None:
+        self.observation_count = int(self.observation_count or 0) + 1
+        self.captured_at = _now_iso()