chendelian
633ecff4e2
Keep the last in-template copy on main so history can restore it; product continues in jiangchang-ui-recorder. Co-authored-by: Cursor <cursoragent@cursor.com>
1221 lines
52 KiB
Python
1221 lines
52 KiB
Python
"""Action recording manager: before → action → after, dual-write v1/v2."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import threading
|
|
import time
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
import html
|
|
import re
|
|
from typing import Any, Callable, Optional
|
|
|
|
from locators.catalog import ElementCatalogService
|
|
from models.action import ACTION_CAPABILITY, ActionFlow, ActionRecord
|
|
from models.serialize import to_dict, write_model_json
|
|
from redaction import redact_input_payload, sanitize_artifact
|
|
from snapshot_service import SnapshotService
|
|
from state_diff import (
|
|
changes_to_dicts,
|
|
diff_states,
|
|
postconditions_to_dicts,
|
|
propose_postconditions,
|
|
)
|
|
from core.policies.element_identity import element_identity_key
|
|
|
|
from .base import RawUserEvent, RecorderState
|
|
from .input_aggregator import InputAggregator
|
|
|
|
|
|
def _now_iso() -> str:
|
|
return datetime.now().astimezone().isoformat()
|
|
|
|
|
|
class ActionRecordingManager:
|
|
"""
|
|
Central coordinator.
|
|
|
|
- Action is the flow spine
|
|
- States are before/after evidence (fingerprint-deduped)
|
|
- Dual-writes optional v1 step via callback
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
flow_dir: Path,
|
|
*,
|
|
flow_name: str = "",
|
|
flow_id: str = "",
|
|
surface: str = "windows",
|
|
recording_mode: str = "auto",
|
|
dump_fn: Optional[Callable[[], Any]] = None,
|
|
write_v1_step: Optional[Callable[..., Any]] = None,
|
|
stability_timeout_sec: float = 3.0,
|
|
save_sensitive_screenshots: bool = False,
|
|
defer_action_evidence: bool = False,
|
|
on_action: Optional[Callable[[ActionRecord], None]] = None,
|
|
on_action_update: Optional[Callable[[ActionRecord, str], None]] = None,
|
|
) -> None:
|
|
self.flow_dir = Path(flow_dir)
|
|
self.actions_dir = self.flow_dir / "actions"
|
|
self.actions_dir.mkdir(parents=True, exist_ok=True)
|
|
self.snapshots = SnapshotService(
|
|
self.flow_dir,
|
|
save_sensitive_screenshots=save_sensitive_screenshots,
|
|
)
|
|
self.catalog = ElementCatalogService(self.flow_dir, flow_id=flow_id or flow_name)
|
|
self.aggregator = InputAggregator(idle_sec=8.0 if surface == "browser" else 0.8)
|
|
self.dump_fn = dump_fn
|
|
self.defer_action_evidence = bool(defer_action_evidence)
|
|
self.on_action = on_action
|
|
self.on_action_update = on_action_update
|
|
self.write_v1_step = write_v1_step
|
|
self.stability_timeout_sec = stability_timeout_sec
|
|
self._lock = threading.RLock()
|
|
self._seq = 0
|
|
self.status = RecorderState.IDLE
|
|
self.last_action: Optional[ActionRecord] = None
|
|
self.last_warning: str = ""
|
|
self.action_flow = ActionFlow(
|
|
flow_id=flow_id or self.flow_dir.name,
|
|
flow_name=flow_name or self.flow_dir.name,
|
|
recording_mode=recording_mode,
|
|
surface=surface,
|
|
metadata={"action_capability": dict(ACTION_CAPABILITY)},
|
|
)
|
|
self._debounce_until = 0.0
|
|
self._debounce_sec = 0.35
|
|
self._last_click_signature = ""
|
|
self._last_click_monotonic = 0.0
|
|
self._actions_by_id: dict[str, ActionRecord] = {}
|
|
self._last_fill: dict[str, tuple[str, float]] = {}
|
|
self._last_text_target: dict[str, Any] = {}
|
|
|
|
def _dump_for_context(self, context: Optional[dict[str, Any]] = None) -> Any:
|
|
if not self.dump_fn:
|
|
return None
|
|
ctx = dict(context or {})
|
|
kwargs = {
|
|
key: ctx.get(key)
|
|
for key in ("page_id", "document_id", "frame_id")
|
|
if ctx.get(key)
|
|
}
|
|
if kwargs:
|
|
try:
|
|
return self.dump_fn(**kwargs)
|
|
except TypeError as exc:
|
|
if "unexpected keyword" not in str(exc) and "positional" not in str(exc):
|
|
raise
|
|
return self.dump_fn()
|
|
|
|
@staticmethod
|
|
def _event_context(event: RawUserEvent) -> dict[str, Any]:
|
|
meta = dict(event.meta or {})
|
|
return {
|
|
"surface": event.surface or "",
|
|
"page_id": meta.get("page_id") or "",
|
|
"document_id": meta.get("document_id") or "",
|
|
"frame_id": meta.get("frame_id") or "",
|
|
"url": event.url or "",
|
|
"frame_url": (event.frame or {}).get("url") or "",
|
|
"frame_name": (event.frame or {}).get("name") or "",
|
|
"source_ts_ms": meta.get("source_ts_ms"),
|
|
"event_id": meta.get("event_id") or "",
|
|
"sequence": meta.get("sequence"),
|
|
"timestamp": event.timestamp,
|
|
"input_event": meta.get("input_event") or "",
|
|
"is_trusted": meta.get("is_trusted"),
|
|
}
|
|
|
|
@staticmethod
|
|
def _captured_live_counts(target: dict[str, Any]) -> Optional[dict[str, int]]:
|
|
"""Use uniqueness measured synchronously in the event-origin document."""
|
|
counts: dict[str, int] = {}
|
|
for strategy, field in (
|
|
("css", "css_count"),
|
|
("stable_css", "stable_css_count"),
|
|
("href", "href_count"),
|
|
):
|
|
value = (target or {}).get(field)
|
|
if value is not None:
|
|
counts[strategy] = int(value)
|
|
return counts or None
|
|
|
|
def _stable_capture(
|
|
self, context: dict[str, Any], *, reason: str, timeout: float
|
|
):
|
|
return self.snapshots.wait_until_stable(
|
|
lambda: self._dump_for_context(context),
|
|
timeout_sec=timeout,
|
|
change_reason=reason,
|
|
)
|
|
|
|
def _action_capture(self, context: dict[str, Any], *, reason: str):
|
|
"""Capture one post-action observation without blocking on two dumps.
|
|
|
|
Browser user events can arrive faster than a full page dump. A second
|
|
stability poll made the FIFO lag by several seconds and contaminated
|
|
earlier actions with later input. The action stream keeps ordering;
|
|
quiet/checkpoint capture remains available for deliberately stable UI.
|
|
"""
|
|
import time
|
|
|
|
time.sleep(0.06)
|
|
dump = self._dump_for_context(context)
|
|
return self.snapshots.capture_from_dump(
|
|
dump, change_reason=reason, force_persist=True
|
|
)
|
|
|
|
@staticmethod
|
|
def _enrich_weak_browser_target(target: dict[str, Any], before) -> dict[str, Any]:
|
|
"""Attach a semantic hint without discarding the physical hit target."""
|
|
out = dict(target or {})
|
|
role = str(out.get("role") or "").lower()
|
|
if role not in {"", "path", "svg", "img", "span", "div"}:
|
|
return out
|
|
if str(out.get("name") or out.get("text") or "").strip():
|
|
return out
|
|
scope_text = " ".join(
|
|
[str(out.get("css") or "")]
|
|
+ [
|
|
" ".join(
|
|
str((ancestor or {}).get(key) or "")
|
|
for key in ("id", "class_name", "css")
|
|
)
|
|
for ancestor in (out.get("parent_chain") or [])
|
|
]
|
|
).lower()
|
|
form_hints = (
|
|
"comment", "editor", "input", "form", "reply", "message", "chat",
|
|
"评论", "输入", "回复", "消息",
|
|
)
|
|
if not any(token in scope_text for token in form_hints):
|
|
return out
|
|
action_names = {
|
|
"submit", "send", "publish", "post", "reply", "comment",
|
|
"提交", "发送", "发布", "回复", "评论",
|
|
"送信", "投稿", "公開", "보내기", "게시", "enviar", "publicar",
|
|
"envoyer", "publier", "senden", "veröffentlichen",
|
|
}
|
|
candidates = []
|
|
for row in list(getattr(before, "locatable", None) or []):
|
|
row_role = str(row.get("role") or row.get("control_type") or "").lower()
|
|
row_name = str(row.get("name") or row.get("value") or row.get("text") or "").strip()
|
|
if row_role in {"button", "menuitem", "link"} and row_name.lower() in action_names:
|
|
candidates.append(row)
|
|
unique = {
|
|
(str(row.get("role") or row.get("control_type") or ""), str(row.get("name") or row.get("value") or row.get("text") or ""))
|
|
for row in candidates
|
|
}
|
|
if len(unique) == 1:
|
|
semantic_role, semantic_name = next(iter(unique))
|
|
out["semantic_role"] = semantic_role
|
|
out["semantic_name"] = semantic_name
|
|
out["semantic_inference"] = "form_scope_unique_action"
|
|
out["semantic_confidence"] = 0.72
|
|
return out
|
|
|
|
def start(self, *, entry_dump: Optional[Any] = None) -> ActionRecord | None:
|
|
with self._lock:
|
|
self.status = RecorderState.RECORDING
|
|
self.action_flow.status = "open"
|
|
self.action_flow.created_at = _now_iso()
|
|
dump = entry_dump
|
|
if dump is None and self.dump_fn:
|
|
try:
|
|
dump = self._dump_for_context()
|
|
except Exception as exc:
|
|
self.last_warning = f"entry dump failed: {exc}"
|
|
dump = None
|
|
if dump is not None:
|
|
st = self.snapshots.capture_from_dump(
|
|
dump, change_reason="entry", force_persist=True
|
|
)
|
|
self.action_flow.entry_state_id = st.id
|
|
self.action_flow.final_state_id = st.id
|
|
if st.id not in self.action_flow.states:
|
|
self.action_flow.states.append(st.id)
|
|
if self.snapshots.latest_observation:
|
|
self.action_flow.observations.append(
|
|
self.snapshots.latest_observation.id
|
|
)
|
|
# Optional v1 step for entry checkpoint
|
|
if self.write_v1_step:
|
|
try:
|
|
self.write_v1_step(dump, label_hint="entry", change_reason="entry")
|
|
except Exception:
|
|
pass
|
|
self._persist_flow()
|
|
return None
|
|
|
|
def pause(self) -> None:
|
|
self.status = RecorderState.PAUSED
|
|
|
|
def resume(self) -> None:
|
|
if self.status == RecorderState.PAUSED:
|
|
self.status = RecorderState.RECORDING
|
|
|
|
def stop(self) -> ActionFlow:
|
|
with self._lock:
|
|
for item in self.aggregator.flush():
|
|
self._commit_fill(item)
|
|
self.status = RecorderState.STOPPED
|
|
self.action_flow.status = "ended"
|
|
self.action_flow.ended_at = _now_iso()
|
|
if self.snapshots.latest:
|
|
self.action_flow.final_state_id = self.snapshots.latest.id
|
|
self._persist_flow()
|
|
self.catalog.save()
|
|
return self.action_flow
|
|
|
|
def handle_raw_event(self, event: RawUserEvent) -> Optional[ActionRecord]:
|
|
if self.status != RecorderState.RECORDING:
|
|
return None
|
|
if event.from_inspector:
|
|
return None
|
|
with self._lock:
|
|
kind = event.kind
|
|
if kind in {"mouse_down", "click"}:
|
|
return self._handle_click(event)
|
|
if kind in {"key", "input", "value_change"}:
|
|
return self._handle_input(event)
|
|
if kind == "navigation":
|
|
return self._handle_navigation(event)
|
|
if kind == "checkpoint":
|
|
return self.record_checkpoint(meta=event.meta)
|
|
return None
|
|
|
|
def record_checkpoint(self, *, meta: Optional[dict[str, Any]] = None) -> Optional[ActionRecord]:
|
|
with self._lock:
|
|
before = self.snapshots.latest
|
|
before_obs = self.snapshots.latest_observation
|
|
dump = None
|
|
if self.dump_fn:
|
|
try:
|
|
dump = self._dump_for_context(meta)
|
|
except Exception as exc:
|
|
return self._failed_action(
|
|
"checkpoint", error=str(exc), before=before
|
|
)
|
|
after = self.snapshots.capture_from_dump(
|
|
dump, change_reason="checkpoint", force_persist=True
|
|
)
|
|
return self._finalize_action(
|
|
action_type="checkpoint",
|
|
before=before,
|
|
after=after,
|
|
target={},
|
|
input_payload={},
|
|
raw_event={"meta": meta or {}},
|
|
confidence=0.9,
|
|
before_observation_id=before_obs.id if before_obs else None,
|
|
after_observation_id=(
|
|
self.snapshots.latest_observation.id
|
|
if self.snapshots.latest_observation else None
|
|
),
|
|
)
|
|
|
|
def _handle_click(self, event: RawUserEvent) -> Optional[ActionRecord]:
|
|
import time
|
|
|
|
now = time.monotonic()
|
|
context = self._event_context(event)
|
|
target = dict(event.target or {})
|
|
click_identity = element_identity_key(
|
|
target,
|
|
surface=event.surface,
|
|
page_url=str(context.get("url") or ""),
|
|
frame_name=str(context.get("frame_name") or ""),
|
|
)
|
|
signature = "|".join(
|
|
(click_identity, str(event.button or ""), str(event.x), str(event.y))
|
|
)
|
|
if (
|
|
signature == self._last_click_signature
|
|
and now - self._last_click_monotonic < self._debounce_sec
|
|
):
|
|
return None
|
|
self._last_click_signature = signature
|
|
self._last_click_monotonic = now
|
|
# Flush pending input before click
|
|
for item in self.aggregator.flush():
|
|
self._commit_fill(item)
|
|
|
|
before = self.snapshots.latest
|
|
before_obs = self.snapshots.latest_observation
|
|
if event.surface == "browser":
|
|
target = self._enrich_weak_browser_target(target, before)
|
|
element = None
|
|
live_counts = self._captured_live_counts(target)
|
|
if (
|
|
not self.defer_action_evidence
|
|
and event.surface == "browser" and context.get("page_id") and target
|
|
):
|
|
try:
|
|
import kit_browser
|
|
|
|
live_counts = kit_browser.validate_element_on_page(
|
|
page_id=str(context.get("page_id") or ""),
|
|
element=target,
|
|
frame=event.frame or None,
|
|
)
|
|
except Exception as exc:
|
|
self.last_warning = f"live locator validation failed: {exc}"
|
|
if target:
|
|
try:
|
|
element = self.catalog.upsert_from_snapshot(
|
|
target,
|
|
surface=event.surface,
|
|
window_scope=target.get("window_scope") or {},
|
|
rows=(before.locatable if before else None),
|
|
frame=event.frame or None,
|
|
live_counts=live_counts,
|
|
)
|
|
except Exception as exc:
|
|
self.last_warning = f"catalog upsert failed: {exc}"
|
|
|
|
after = before
|
|
status = "ok"
|
|
err = ""
|
|
if self.dump_fn and not self.defer_action_evidence:
|
|
try:
|
|
after = self._action_capture(context, reason="after_click")
|
|
except Exception as exc:
|
|
status = "partial"
|
|
err = str(exc)
|
|
try:
|
|
dump = self._dump_for_context(context)
|
|
after = self.snapshots.capture_from_dump(
|
|
dump, change_reason="after_click_partial", force_persist=True
|
|
)
|
|
except Exception as exc2:
|
|
status = "failed"
|
|
err = f"{err}; {exc2}"
|
|
|
|
act = self._finalize_action(
|
|
action_type="click",
|
|
before=before,
|
|
after=after,
|
|
target=target,
|
|
target_element_id=element.id if element else None,
|
|
input_payload={},
|
|
raw_event=to_dict(event) if hasattr(event, "__dataclass_fields__") else dict(event.__dict__),
|
|
status=status,
|
|
confidence=0.8 if status == "ok" else 0.4,
|
|
evidence={"error": err} if err else {},
|
|
started_at=event.timestamp,
|
|
before_observation_id=before_obs.id if before_obs else None,
|
|
after_observation_id=(
|
|
self.snapshots.latest_observation.id
|
|
if self.snapshots.latest_observation else None
|
|
),
|
|
)
|
|
if self.write_v1_step and after is not None:
|
|
# Dual-write one v1 step per meaningful action (not heartbeat)
|
|
try:
|
|
# Reconstruct a minimal dump-like for v1 from after state paths is hard;
|
|
# caller should pass dump through write_v1_step if needed.
|
|
pass
|
|
except Exception:
|
|
pass
|
|
return act
|
|
|
|
def _handle_input(self, event: RawUserEvent) -> Optional[ActionRecord]:
|
|
key = event.key or (event.meta or {}).get("key") or ""
|
|
text = event.text or (event.meta or {}).get("text") or (event.meta or {}).get("value") or ""
|
|
common = {
|
|
"target": event.target,
|
|
"input_type": str((event.meta or {}).get("input_type") or ""),
|
|
"field_name": str((event.target or {}).get("name") or ""),
|
|
"capture_context": self._event_context(event),
|
|
"raw_event": to_dict(event),
|
|
}
|
|
source_ms = (event.meta or {}).get("source_ts_ms")
|
|
event_clock = (
|
|
float(source_ms) / 1000.0 if source_ms is not None else time.monotonic()
|
|
)
|
|
if event.kind in {"input", "value_change"}:
|
|
items = self.aggregator.on_value_change(value=text, now=event_clock, **common)
|
|
else:
|
|
items = self.aggregator.on_key(key=key, text=text, now=event_clock, **common)
|
|
items.extend(self.aggregator.maybe_flush_idle(now=event_clock))
|
|
last = None
|
|
for item in items:
|
|
if item.action_type == "press_key":
|
|
last = self._commit_press_key(item)
|
|
else:
|
|
last = self._commit_fill(item)
|
|
return last
|
|
|
|
def _handle_navigation(self, event: RawUserEvent) -> Optional[ActionRecord]:
|
|
before = self.snapshots.latest
|
|
before_obs = self.snapshots.latest_observation
|
|
context = self._event_context(event)
|
|
after = before
|
|
if self.dump_fn and not self.defer_action_evidence:
|
|
try:
|
|
after = self._action_capture(context, reason="navigation")
|
|
except Exception:
|
|
pass
|
|
return self._finalize_action(
|
|
action_type="navigation",
|
|
before=before,
|
|
after=after,
|
|
target={},
|
|
input_payload={"url": event.url},
|
|
raw_event={"url": event.url, "meta": event.meta},
|
|
confidence=0.7,
|
|
started_at=event.timestamp,
|
|
before_observation_id=before_obs.id if before_obs else None,
|
|
after_observation_id=(self.snapshots.latest_observation.id if self.snapshots.latest_observation else None),
|
|
)
|
|
|
|
def _commit_fill(self, item) -> Optional[ActionRecord]:
|
|
if (
|
|
item.target
|
|
and (self.action_flow.surface or "") == "windows"
|
|
and not self._is_oversized_windows_target(item.target)
|
|
):
|
|
self._last_text_target = dict(item.target)
|
|
page_url = str(item.capture_context.get("url") or "")
|
|
identity = element_identity_key(
|
|
item.target or {},
|
|
surface=self.action_flow.surface or "windows",
|
|
page_url=page_url,
|
|
frame_name=str(item.capture_context.get("frame_name") or ""),
|
|
)
|
|
text_value = str((item.input_payload or {}).get("text") or "")
|
|
previous = self._last_fill.get(identity)
|
|
source_ms = item.capture_context.get("source_ts_ms")
|
|
now_marker = float(source_ms) / 1000.0 if source_ms else time.time()
|
|
if previous and previous[0] == text_value and now_marker - previous[1] <= 15.0:
|
|
return None
|
|
self._last_fill[identity] = (text_value, now_marker)
|
|
before = self.snapshots.latest
|
|
before_obs = self.snapshots.latest_observation
|
|
after = before
|
|
if self.dump_fn and not self.defer_action_evidence:
|
|
try:
|
|
after = self._action_capture(
|
|
item.capture_context, reason="after_fill"
|
|
)
|
|
except Exception:
|
|
pass
|
|
element = None
|
|
frame = {
|
|
"url": item.capture_context.get("frame_url") or "",
|
|
"name": item.capture_context.get("frame_name") or "",
|
|
}
|
|
live_counts = self._captured_live_counts(item.target or {})
|
|
if (
|
|
not self.defer_action_evidence
|
|
and item.capture_context.get("page_id") and item.target
|
|
):
|
|
try:
|
|
import kit_browser
|
|
|
|
live_counts = kit_browser.validate_element_on_page(
|
|
page_id=str(item.capture_context.get("page_id") or ""),
|
|
element=item.target,
|
|
frame=frame,
|
|
)
|
|
except Exception as exc:
|
|
self.last_warning = f"live fill locator validation failed: {exc}"
|
|
if item.target:
|
|
try:
|
|
element = self.catalog.upsert_from_snapshot(
|
|
item.target,
|
|
surface=self.action_flow.surface or "windows",
|
|
window_scope=item.target.get("window_scope") or {},
|
|
rows=(before.locatable if before else None),
|
|
frame=frame,
|
|
live_counts=live_counts,
|
|
)
|
|
except Exception:
|
|
pass
|
|
return self._finalize_action(
|
|
action_type="fill",
|
|
before=before,
|
|
after=after,
|
|
target=item.target,
|
|
target_element_id=element.id if element else None,
|
|
input_payload=item.input_payload,
|
|
sensitive=item.sensitive,
|
|
confidence=0.75,
|
|
raw_event=item.raw_event or {
|
|
"url": item.capture_context.get("url") or "",
|
|
"meta": dict(item.capture_context or {}),
|
|
},
|
|
raw_event_ids=item.raw_event_ids,
|
|
started_at=str(item.capture_context.get("timestamp") or _now_iso()),
|
|
before_observation_id=before_obs.id if before_obs else None,
|
|
after_observation_id=(self.snapshots.latest_observation.id if self.snapshots.latest_observation else None),
|
|
)
|
|
|
|
def _commit_press_key(self, item) -> ActionRecord:
|
|
target = dict(item.target or {})
|
|
if (
|
|
(self.action_flow.surface or "") == "windows"
|
|
and self._is_oversized_windows_target(target)
|
|
and self._last_text_target
|
|
and self._same_windows_target_scope(target, self._last_text_target)
|
|
):
|
|
target = dict(self._last_text_target)
|
|
before = self.snapshots.latest
|
|
before_obs = self.snapshots.latest_observation
|
|
after = before
|
|
if self.dump_fn and not self.defer_action_evidence:
|
|
try:
|
|
after = self._action_capture(
|
|
item.capture_context, reason="after_key"
|
|
)
|
|
except Exception:
|
|
pass
|
|
element = None
|
|
frame = {
|
|
"url": item.capture_context.get("frame_url") or "",
|
|
"name": item.capture_context.get("frame_name") or "",
|
|
}
|
|
live_counts = self._captured_live_counts(target)
|
|
if target:
|
|
try:
|
|
element = self.catalog.upsert_from_snapshot(
|
|
target,
|
|
surface=self.action_flow.surface or "windows",
|
|
window_scope=target.get("window_scope") or {},
|
|
rows=(before.locatable if before else None),
|
|
frame=frame,
|
|
live_counts=live_counts,
|
|
)
|
|
except Exception as exc:
|
|
self.last_warning = f"key target catalog upsert failed: {exc}"
|
|
return self._finalize_action(
|
|
action_type="press_key",
|
|
before=before,
|
|
after=after,
|
|
target=target,
|
|
target_element_id=element.id if element else None,
|
|
input_payload=item.input_payload,
|
|
sensitive=item.sensitive,
|
|
confidence=0.7,
|
|
raw_event=item.raw_event or {
|
|
"kind": "key",
|
|
"surface": self.action_flow.surface or "",
|
|
"key": item.key,
|
|
"target": target,
|
|
"frame": frame,
|
|
"url": item.capture_context.get("url") or "",
|
|
"meta": dict(item.capture_context or {}),
|
|
},
|
|
raw_event_ids=item.raw_event_ids,
|
|
started_at=str(item.capture_context.get("timestamp") or _now_iso()),
|
|
before_observation_id=before_obs.id if before_obs else None,
|
|
after_observation_id=(self.snapshots.latest_observation.id if self.snapshots.latest_observation else None),
|
|
)
|
|
|
|
def _failed_action(
|
|
self, action_type: str, *, error: str, before
|
|
) -> ActionRecord:
|
|
return self._finalize_action(
|
|
action_type=action_type,
|
|
before=before,
|
|
after=before,
|
|
target={},
|
|
input_payload={},
|
|
status="failed",
|
|
confidence=0.1,
|
|
evidence={"error": error},
|
|
)
|
|
|
|
def _finalize_action(
|
|
self,
|
|
*,
|
|
action_type: str,
|
|
before,
|
|
after,
|
|
target: dict[str, Any],
|
|
input_payload: dict[str, Any],
|
|
target_element_id: Optional[str] = None,
|
|
sensitive: bool = False,
|
|
raw_event: Optional[dict[str, Any]] = None,
|
|
raw_event_ids: Optional[list[str]] = None,
|
|
status: str = "ok",
|
|
confidence: float = 0.5,
|
|
evidence: Optional[dict[str, Any]] = None,
|
|
before_observation_id: Optional[str] = None,
|
|
after_observation_id: Optional[str] = None,
|
|
started_at: Optional[str] = None,
|
|
) -> ActionRecord:
|
|
self._seq += 1
|
|
payload, sens2 = redact_input_payload(input_payload or {})
|
|
sensitive = sensitive or sens2
|
|
clean_target = sanitize_artifact(target or {}, sensitive_hint=sensitive)
|
|
clean_raw_event = sanitize_artifact(raw_event or {}, sensitive_hint=sensitive)
|
|
|
|
before_id = before.id if before else None
|
|
after_id = after.id if after else before_id
|
|
if self.defer_action_evidence and action_type != "checkpoint":
|
|
# A slow eventual dump must not be presented as an event-time
|
|
# after-state. The evidence coordinator attaches it explicitly.
|
|
after_id = None
|
|
after_observation_id = None
|
|
if after and after.id not in self.action_flow.states:
|
|
self.action_flow.states.append(after.id)
|
|
if before and before.id not in self.action_flow.states:
|
|
self.action_flow.states.append(before.id)
|
|
|
|
observed = []
|
|
proposed = []
|
|
if before and after:
|
|
b = self.snapshots.to_summary_dict(before)
|
|
a = self.snapshots.to_summary_dict(after)
|
|
changes = diff_states(b, a)
|
|
observed = changes_to_dicts(changes)
|
|
proposed = postconditions_to_dicts(propose_postconditions(changes))
|
|
|
|
# Attach locator uniqueness hint for UI
|
|
match_count = None
|
|
unique = None
|
|
primary_expr = ""
|
|
if target_element_id:
|
|
for el in self.catalog.catalog.elements:
|
|
if el.id == target_element_id and el.primary_locator:
|
|
match_count = el.primary_locator.actual_count
|
|
unique = el.primary_locator.unique
|
|
primary_expr = el.primary_locator.expression
|
|
break
|
|
if target_element_id not in self.action_flow.elements:
|
|
self.action_flow.elements.append(target_element_id)
|
|
|
|
if raw_event_ids is None:
|
|
event_id = str(((clean_raw_event or {}).get("meta") or {}).get("event_id") or "")
|
|
raw_event_ids = [event_id] if event_id else []
|
|
act = ActionRecord(
|
|
sequence=self._seq,
|
|
action_type=action_type,
|
|
started_at=started_at or _now_iso(),
|
|
ended_at=_now_iso(),
|
|
surface=self.action_flow.surface or "",
|
|
target_element_id=target_element_id,
|
|
target_snapshot=dict(clean_target or {}),
|
|
input=payload,
|
|
sensitive=sensitive,
|
|
before_state_id=before_id,
|
|
after_state_id=after_id,
|
|
before_observation_id=before_observation_id,
|
|
after_observation_id=after_observation_id,
|
|
observed_changes=observed,
|
|
proposed_postconditions=proposed,
|
|
status=status,
|
|
confidence=confidence,
|
|
evidence={
|
|
**(evidence or {}),
|
|
"locator_match_count": match_count,
|
|
"locator_unique": unique,
|
|
"primary_locator": primary_expr,
|
|
},
|
|
raw_event=dict(clean_raw_event or {}),
|
|
raw_event_ids=list(raw_event_ids or []),
|
|
)
|
|
write_model_json(self.actions_dir / f"{act.id}.json", act)
|
|
self._actions_by_id[act.id] = act
|
|
self.action_flow.actions.append(act.id)
|
|
for observation_id in (before_observation_id, after_observation_id):
|
|
if observation_id and observation_id not in self.action_flow.observations:
|
|
self.action_flow.observations.append(observation_id)
|
|
if after_id:
|
|
self.action_flow.final_state_id = after_id
|
|
self.last_action = act
|
|
self._persist_flow()
|
|
self._notify_action_update(act, "captured")
|
|
if self.on_action is not None and action_type != "checkpoint":
|
|
try:
|
|
self.on_action(act)
|
|
except Exception as exc:
|
|
self.last_warning = f"action evidence scheduling failed: {exc}"
|
|
# Dual-write optional pointer fields into flow.json via side file is enough;
|
|
# artifacts integration updates flow.json separately.
|
|
return act
|
|
|
|
def attach_eventual_evidence(
|
|
self,
|
|
action_ids: list[str],
|
|
*,
|
|
dump: Any,
|
|
requested_monotonic: float,
|
|
) -> None:
|
|
"""Attach one coalesced full dump without rewriting causal after-state."""
|
|
with self._lock:
|
|
lag_ms = max(0, int((time.monotonic() - requested_monotonic) * 1000))
|
|
state = self.snapshots.capture_from_dump(
|
|
dump, change_reason="eventual_after_actions", force_persist=True
|
|
)
|
|
observation = self.snapshots.latest_observation
|
|
if state.id not in self.action_flow.states:
|
|
self.action_flow.states.append(state.id)
|
|
if observation and observation.id not in self.action_flow.observations:
|
|
self.action_flow.observations.append(observation.id)
|
|
self.action_flow.final_state_id = state.id
|
|
if (self.action_flow.surface or "") == "windows":
|
|
self._promote_windows_targets_from_dump(
|
|
dump, action_ids=action_ids, evidence_lag_ms=lag_ms
|
|
)
|
|
for action_id in action_ids:
|
|
action = self._actions_by_id.get(action_id)
|
|
if action is None:
|
|
continue
|
|
action.evidence.update(
|
|
{
|
|
"eventual_observation_id": observation.id if observation else "",
|
|
"eventual_state_id": state.id,
|
|
"evidence_association": "coalesced_eventual",
|
|
"evidence_lag_ms": lag_ms,
|
|
}
|
|
)
|
|
write_model_json(self.actions_dir / f"{action.id}.json", action)
|
|
self._notify_action_update(action, "evidence")
|
|
self._persist_flow()
|
|
|
|
@staticmethod
|
|
def _is_oversized_windows_target(target: dict[str, Any]) -> bool:
|
|
bounds = dict(target.get("bounds") or target.get("rect") or {})
|
|
area = int(bounds.get("width") or 0) * int(bounds.get("height") or 0)
|
|
return str(target.get("control_type") or "") == "WindowControl" or area > 1_000_000
|
|
|
|
@staticmethod
|
|
def _is_broad_windows_target(target: dict[str, Any]) -> bool:
|
|
if ActionRecordingManager._is_oversized_windows_target(target):
|
|
return True
|
|
bounds = dict(target.get("bounds") or target.get("rect") or {})
|
|
area = int(bounds.get("width") or 0) * int(bounds.get("height") or 0)
|
|
control_type = str(target.get("control_type") or "")
|
|
name = str(target.get("name") or target.get("automation_id") or "").lower()
|
|
layout_tokens = (
|
|
"search_result", "listview", "list_view", "watermark",
|
|
"container", "layout", "clientbg", "panel", "pane",
|
|
)
|
|
return (
|
|
area > 250_000
|
|
or (
|
|
control_type in {"TreeControl", "PaneControl", "GroupControl"}
|
|
and any(token in name for token in layout_tokens)
|
|
)
|
|
)
|
|
|
|
@staticmethod
|
|
def _unique_windows_row_candidate(
|
|
rows: list[dict[str, Any]],
|
|
) -> Optional[dict[str, Any]]:
|
|
"""Return a single visible result/list row when geometry is unreliable."""
|
|
row_tokens = (
|
|
"user_item", "result_item", "search_item", "contact_item",
|
|
"list_item", "listitem", "result_row", "contact_row",
|
|
)
|
|
candidates: dict[tuple[Any, ...], tuple[int, dict[str, Any]]] = {}
|
|
for index, row in enumerate(rows):
|
|
if row.get("is_offscreen") is True:
|
|
continue
|
|
control_type = str(row.get("control_type") or "")
|
|
name = " ".join(
|
|
str(row.get(key) or "")
|
|
for key in ("name", "automation_id", "class_name")
|
|
).lower()
|
|
if not (
|
|
any(token in name for token in row_tokens)
|
|
or control_type in {"ListItemControl", "DataItemControl"}
|
|
):
|
|
continue
|
|
bounds = dict(row.get("bounds") or row.get("rect") or {})
|
|
width = int(bounds.get("width") or 0)
|
|
height = int(bounds.get("height") or 0)
|
|
if width < 8 or height < 8:
|
|
continue
|
|
key = (
|
|
int(bounds.get("left") or 0), int(bounds.get("top") or 0),
|
|
int(bounds.get("right") or 0), int(bounds.get("bottom") or 0),
|
|
str(row.get("name") or ""), str(row.get("value") or ""),
|
|
)
|
|
candidates[key] = (index, dict(row))
|
|
if len(candidates) != 1:
|
|
return None
|
|
index, chosen = next(iter(candidates.values()))
|
|
chosen["bounds"] = dict(chosen.get("bounds") or chosen.get("rect") or {})
|
|
chosen.pop("rect", None)
|
|
# A row container often has no accessible value of its own. Prefer an
|
|
# explicit row-name descendant over structural labels such as
|
|
# search_avatar_label, and strip WeCom rich-text markup.
|
|
row_depth = int(chosen.get("depth") or 0)
|
|
structural = {
|
|
"layout", "container", "controlui", "horizontal", "vertical",
|
|
"user_item", "list_item", "result_item",
|
|
}
|
|
descendants: list[tuple[int, str, str, dict[str, Any]]] = []
|
|
preferred_tokens = (
|
|
"search_name", "groupbuddyname", "contact_name", "display_name",
|
|
"result_name", "title_label",
|
|
)
|
|
rejected_tokens = ("avatar", "layout", "container", "controlui")
|
|
for child in rows[index + 1 :]:
|
|
child_depth = int(child.get("depth") or 0)
|
|
if child_depth <= row_depth:
|
|
break
|
|
raw_value = str(child.get("value") or "").strip()
|
|
name = str(child.get("name") or "").strip()
|
|
clean_value = html.unescape(re.sub(r"<[^>]+>", "", raw_value)).strip()
|
|
lowered = name.lower()
|
|
semantic = clean_value or (
|
|
name if name and not any(token in lowered for token in structural) else ""
|
|
)
|
|
if not semantic or semantic == "[REDACTED]" or any(
|
|
token in lowered for token in rejected_tokens
|
|
):
|
|
continue
|
|
rank = 0 if any(token in lowered for token in preferred_tokens) else 1
|
|
descendants.append((rank, semantic, name, child))
|
|
if descendants:
|
|
_rank, semantic, name, child = min(descendants, key=lambda item: item[0])
|
|
chosen["semantic_name"] = semantic
|
|
chosen["row_descendant"] = {
|
|
"name": name,
|
|
"value": str(child.get("value") or "").strip(),
|
|
"control_type": child.get("control_type") or "",
|
|
}
|
|
if not str(chosen.get("value") or ""):
|
|
chosen["value"] = semantic
|
|
return chosen
|
|
|
|
@staticmethod
|
|
def _windows_hit_candidate(
|
|
rows: list[dict[str, Any]], x: int, y: int
|
|
) -> Optional[dict[str, Any]]:
|
|
interactive = {
|
|
"ButtonControl", "EditControl", "TreeItemControl", "ListItemControl",
|
|
"MenuItemControl", "CheckBoxControl", "RadioButtonControl",
|
|
"ComboBoxControl", "TabItemControl", "HyperlinkControl",
|
|
}
|
|
semantic_tokens = (
|
|
"search", "searchedit", "edit", "input", "message", "send",
|
|
"comment", "reply", "搜索", "输入", "发送", "消息",
|
|
)
|
|
candidates: list[tuple[tuple[int, int, int, int], dict[str, Any]]] = []
|
|
for row in rows:
|
|
bounds = dict(row.get("bounds") or row.get("rect") or {})
|
|
try:
|
|
inside = (
|
|
int(bounds.get("left")) <= x <= int(bounds.get("right"))
|
|
and int(bounds.get("top")) <= y <= int(bounds.get("bottom"))
|
|
)
|
|
except (TypeError, ValueError):
|
|
continue
|
|
if not inside or row.get("is_offscreen") is True:
|
|
continue
|
|
width = max(0, int(bounds.get("width") or 0))
|
|
height = max(0, int(bounds.get("height") or 0))
|
|
area = width * height
|
|
control_type = str(row.get("control_type") or "")
|
|
# Hairlines, separators and layout-only artifacts are never useful
|
|
# click targets even when they happen to be the smallest hit.
|
|
if (
|
|
not area
|
|
or width < 4
|
|
or height < 4
|
|
or control_type == "WindowControl"
|
|
):
|
|
continue
|
|
text = " ".join(
|
|
str(row.get(key) or "")
|
|
for key in ("name", "value", "automation_id", "class_name")
|
|
).lower()
|
|
if any(token in text for token in ("separator", "seperator", "divider", "splitter")):
|
|
continue
|
|
# A named editor/result is stronger than a smaller generic list
|
|
# item covering the same point.
|
|
exact_editor = any(
|
|
token in text
|
|
for token in ("searchedit", "send_rich_edit", "compose_box", "search_box")
|
|
) and "_layout" not in text
|
|
if exact_editor:
|
|
priority = 0
|
|
elif any(token in text for token in semantic_tokens):
|
|
priority = 1
|
|
elif control_type in interactive:
|
|
priority = 2
|
|
else:
|
|
priority = 3
|
|
layout_penalty = 1 if "layout" in text or "container" in text else 0
|
|
redacted = 1 if "[redacted]" in text else 0
|
|
candidates.append(((priority, layout_penalty, redacted, area), row))
|
|
if not candidates:
|
|
return None
|
|
chosen = dict(min(candidates, key=lambda item: item[0])[1])
|
|
bounds = dict(chosen.get("bounds") or chosen.get("rect") or {})
|
|
chosen["bounds"] = bounds
|
|
chosen.pop("rect", None)
|
|
chosen["capture_point"] = {"x": x, "y": y}
|
|
return chosen
|
|
|
|
@staticmethod
|
|
def _windows_candidate_quality(target: dict[str, Any]) -> tuple[int, int, int]:
|
|
bounds = dict(target.get("bounds") or target.get("rect") or {})
|
|
area = int(bounds.get("width") or 0) * int(bounds.get("height") or 0)
|
|
control_type = str(target.get("control_type") or "")
|
|
text = " ".join(
|
|
str(target.get(key) or "")
|
|
for key in ("name", "value", "automation_id", "class_name")
|
|
).lower()
|
|
interactive = control_type in {
|
|
"ButtonControl", "EditControl", "TreeItemControl", "ListItemControl",
|
|
"MenuItemControl", "CheckBoxControl", "RadioButtonControl",
|
|
"ComboBoxControl", "TabItemControl", "HyperlinkControl",
|
|
}
|
|
semantic = any(
|
|
token in text
|
|
for token in ("search", "edit", "input", "message", "send", "buddy", "result")
|
|
)
|
|
exact_editor = any(
|
|
token in text
|
|
for token in ("searchedit", "send_rich_edit", "compose_box", "search_box")
|
|
) and "_layout" not in text
|
|
return (2 if exact_editor else (1 if semantic else 0), 1 if interactive else 0, -area)
|
|
|
|
@staticmethod
|
|
def _clean_windows_rich_text(value: str) -> str:
|
|
return html.unescape(re.sub(r"<[^>]+>", "", str(value or ""))).strip()
|
|
|
|
@staticmethod
|
|
def _same_windows_target_scope(
|
|
first: dict[str, Any], second: dict[str, Any]
|
|
) -> bool:
|
|
a = dict(first.get("window_scope") or {})
|
|
b = dict(second.get("window_scope") or {})
|
|
ah, bh = int(a.get("hwnd") or 0), int(b.get("hwnd") or 0)
|
|
if ah and bh:
|
|
return ah == bh
|
|
ap, bp = int(a.get("pid") or 0), int(b.get("pid") or 0)
|
|
ac = str(a.get("class_name") or first.get("class_name") or "")
|
|
bc = str(b.get("class_name") or second.get("class_name") or "")
|
|
return bool(ap and ap == bp and (not ac or not bc or ac == bc))
|
|
|
|
@staticmethod
|
|
def _windows_scope_key(target: dict[str, Any]) -> tuple[Any, ...]:
|
|
scope = dict(target.get("window_scope") or {})
|
|
hwnd = int(scope.get("hwnd") or 0)
|
|
if hwnd:
|
|
return ("hwnd", hwnd)
|
|
return (
|
|
"process",
|
|
int(scope.get("pid") or 0),
|
|
str(scope.get("class_name") or target.get("class_name") or ""),
|
|
)
|
|
|
|
@staticmethod
|
|
def _is_dynamic_windows_candidate(target: dict[str, Any]) -> bool:
|
|
name = str(target.get("name") or target.get("automation_id") or "").lower()
|
|
value = str(target.get("value") or "").strip()
|
|
dynamic_tokens = (
|
|
"description", "preview", "snippet", "latest_message",
|
|
"last_message", "timestamp", "time_label", "消息摘要", "最新消息",
|
|
)
|
|
return bool(value and any(token in name for token in dynamic_tokens))
|
|
|
|
@staticmethod
|
|
def _same_windows_scope(target: dict[str, Any], window: dict[str, Any]) -> bool:
|
|
scope = dict(target.get("window_scope") or {})
|
|
target_hwnd = int(scope.get("hwnd") or 0)
|
|
dump_hwnd = int(window.get("hwnd") or 0)
|
|
if target_hwnd and dump_hwnd:
|
|
return target_hwnd == dump_hwnd
|
|
target_pid = int(scope.get("pid") or 0)
|
|
dump_pid = int(window.get("pid") or 0)
|
|
target_class = str(scope.get("class_name") or target.get("class_name") or "")
|
|
dump_class = str(window.get("class_name") or "")
|
|
if not scope:
|
|
# Synthetic/unit captures may only carry the root class. Real
|
|
# Windows events include hwnd/pid; class equality is the safest
|
|
# compatibility fallback when that identity is unavailable.
|
|
return bool(target_class and target_class == dump_class)
|
|
return bool(target_pid and target_pid == dump_pid and target_class == dump_class)
|
|
|
|
def _promote_windows_targets_from_dump(
|
|
self,
|
|
dump: Any,
|
|
*,
|
|
action_ids: Optional[list[str]] = None,
|
|
evidence_lag_ms: int = 0,
|
|
) -> None:
|
|
rows = list(getattr(dump, "locatable", None) or [])
|
|
window = dict(getattr(dump, "window", None) or {})
|
|
if not rows:
|
|
return
|
|
last_business_by_scope: dict[tuple[Any, ...], dict[str, Any]] = {}
|
|
allowed = set(action_ids or [])
|
|
actions = sorted(self._actions_by_id.values(), key=lambda item: item.sequence)
|
|
for action in actions:
|
|
target = dict(action.target_snapshot or {})
|
|
scope_key = self._windows_scope_key(target)
|
|
should_update = not allowed or action.id in allowed
|
|
if not should_update:
|
|
if target and not self._is_broad_windows_target(target):
|
|
last_business_by_scope[scope_key] = target
|
|
continue
|
|
# Eventual evidence from another application or from WeCom's main
|
|
# window must never rewrite a transient popup/screenshot action.
|
|
if not self._same_windows_scope(target, window):
|
|
continue
|
|
promoted: Optional[dict[str, Any]] = None
|
|
promotion_reason = "eventual_tree_coordinate_hit"
|
|
if action.action_type == "click" and self._is_broad_windows_target(target):
|
|
raw = dict(action.raw_event or {})
|
|
meta = dict(raw.get("meta") or {})
|
|
point = dict(meta.get("uia_coordinates") or {})
|
|
points: list[tuple[int, int]] = []
|
|
x = point.get("x", raw.get("x"))
|
|
y = point.get("y", raw.get("y"))
|
|
if x is not None and y is not None:
|
|
points.append((int(x), int(y)))
|
|
for candidate in meta.get("uia_coordinate_candidates") or []:
|
|
cx, cy = candidate.get("x"), candidate.get("y")
|
|
if cx is not None and cy is not None:
|
|
p = (int(cx), int(cy))
|
|
if p not in points:
|
|
points.append(p)
|
|
hits = [
|
|
hit
|
|
for px, py in points
|
|
for hit in [self._windows_hit_candidate(rows, px, py)]
|
|
if hit is not None
|
|
]
|
|
if hits:
|
|
promoted = max(hits, key=self._windows_candidate_quality)
|
|
window_class = str(window.get("class_name") or "")
|
|
if (
|
|
(not promoted or self._is_broad_windows_target(promoted))
|
|
and window_class == "SearchResultWindow2"
|
|
):
|
|
unique_row = self._unique_windows_row_candidate(rows)
|
|
if unique_row:
|
|
promoted = unique_row
|
|
promotion_reason = "eventual_tree_unique_row"
|
|
elif action.action_type in {"fill", "type", "press_key"} and self._is_oversized_windows_target(target):
|
|
promoted = dict(last_business_by_scope.get(scope_key) or {}) or None
|
|
elif target and not self._is_broad_windows_target(target):
|
|
last_business_by_scope[scope_key] = target
|
|
|
|
if not promoted:
|
|
continue
|
|
if evidence_lag_ms > 5_000 and self._is_dynamic_windows_candidate(promoted):
|
|
action.evidence.update(
|
|
{
|
|
"target_promotion_rejected": "late_dynamic_candidate",
|
|
"rejected_candidate_name": promoted.get("name") or "",
|
|
"rejected_evidence_lag_ms": int(evidence_lag_ms),
|
|
}
|
|
)
|
|
write_model_json(self.actions_dir / f"{action.id}.json", action)
|
|
continue
|
|
promoted["window_scope"] = dict(
|
|
promoted.get("window_scope") or target.get("window_scope") or window
|
|
)
|
|
promoted_name = str(promoted.get("name") or "").lower()
|
|
if any(
|
|
token in promoted_name
|
|
for token in ("search_name", "contact_name", "groupbuddyname", "display_name")
|
|
):
|
|
cleaned_value = self._clean_windows_rich_text(
|
|
str(promoted.get("value") or "")
|
|
)
|
|
if cleaned_value:
|
|
promoted["value"] = cleaned_value
|
|
promoted["semantic_name"] = cleaned_value
|
|
element = self.catalog.upsert_from_snapshot(
|
|
promoted,
|
|
surface="windows",
|
|
window_scope=promoted.get("window_scope") or {},
|
|
rows=rows,
|
|
)
|
|
action.target_snapshot = sanitize_artifact(promoted)
|
|
action.target_element_id = element.id
|
|
action.confidence = max(float(action.confidence or 0.0), 0.72)
|
|
action.evidence.update(
|
|
{
|
|
"target_promotion": promotion_reason,
|
|
"original_target_control_type": target.get("control_type") or "",
|
|
"locator_match_count": (
|
|
element.primary_locator.actual_count if element.primary_locator else None
|
|
),
|
|
"locator_unique": (
|
|
element.primary_locator.unique if element.primary_locator else None
|
|
),
|
|
"primary_locator": (
|
|
element.primary_locator.expression if element.primary_locator else ""
|
|
),
|
|
}
|
|
)
|
|
if isinstance(action.raw_event, dict):
|
|
action.raw_event["target"] = sanitize_artifact(promoted)
|
|
if element.id not in self.action_flow.elements:
|
|
self.action_flow.elements.append(element.id)
|
|
last_business_by_scope[scope_key] = promoted
|
|
write_model_json(self.actions_dir / f"{action.id}.json", action)
|
|
|
|
def note_evidence_failure(self, action_ids: list[str], error: str) -> None:
|
|
with self._lock:
|
|
self.last_warning = f"eventual evidence failed: {error}"
|
|
for action_id in action_ids:
|
|
action = self._actions_by_id.get(action_id)
|
|
if action is None:
|
|
continue
|
|
action.evidence["eventual_evidence_error"] = str(error)
|
|
write_model_json(self.actions_dir / f"{action.id}.json", action)
|
|
self._notify_action_update(action, "evidence_failed")
|
|
|
|
def _notify_action_update(self, action: ActionRecord, phase: str) -> None:
|
|
if self.on_action_update is None:
|
|
return
|
|
try:
|
|
self.on_action_update(action, phase)
|
|
except Exception as exc:
|
|
self.last_warning = f"action UI update failed: {exc}"
|
|
|
|
def _persist_flow(self) -> None:
|
|
write_model_json(self.flow_dir / "action_flow.json", self.action_flow)
|
|
|
|
def heartbeat(self) -> None:
|
|
"""Lightweight monitor: flush idle input; do NOT create states/actions."""
|
|
if self.status != RecorderState.RECORDING:
|
|
return
|
|
with self._lock:
|
|
for item in self.aggregator.maybe_flush_idle():
|
|
if item.action_type == "press_key":
|
|
self._commit_press_key(item)
|
|
else:
|
|
self._commit_fill(item)
|
|
|
|
def stats(self) -> dict[str, Any]:
|
|
la = self.last_action
|
|
return {
|
|
"recorder_status": self.status,
|
|
"state_count": len(self.action_flow.states),
|
|
"observation_count": len(self.snapshots.observation_ids()),
|
|
"action_count": len(self.action_flow.actions),
|
|
"element_count": len(self.catalog.catalog.elements),
|
|
"last_action_type": la.action_type if la else "",
|
|
"last_target": (la.target_snapshot or {}).get("name")
|
|
or (la.target_snapshot or {}).get("automation_id")
|
|
or (la.target_snapshot or {}).get("semantic_name")
|
|
or (la.target_snapshot or {}).get("class_name")
|
|
or (la.target_snapshot or {}).get("tag")
|
|
if la
|
|
else "",
|
|
"last_match_count": (la.evidence or {}).get("locator_match_count") if la else None,
|
|
"last_unique": (la.evidence or {}).get("locator_unique") if la else None,
|
|
"last_page_id": ((la.raw_event or {}).get("meta") or {}).get("page_id") if la else "",
|
|
"last_observation_id": (
|
|
la.after_observation_id
|
|
or (la.evidence or {}).get("eventual_observation_id")
|
|
if la else ""
|
|
),
|
|
"warning": self.last_warning,
|
|
"recording_mode": self.action_flow.recording_mode,
|
|
}
|