"""Action recording manager: before → action → after, dual-write v1/v2.""" from __future__ import annotations import threading import time from datetime import datetime from pathlib import Path import html import re from typing import Any, Callable, Optional from locators.catalog import ElementCatalogService from models.action import ACTION_CAPABILITY, ActionFlow, ActionRecord from models.serialize import to_dict, write_model_json from redaction import redact_input_payload, sanitize_artifact from snapshot_service import SnapshotService from state_diff import ( changes_to_dicts, diff_states, postconditions_to_dicts, propose_postconditions, ) from core.policies.element_identity import element_identity_key from .base import RawUserEvent, RecorderState from .input_aggregator import InputAggregator def _now_iso() -> str: return datetime.now().astimezone().isoformat() class ActionRecordingManager: """ Central coordinator. - Action is the flow spine - States are before/after evidence (fingerprint-deduped) - Dual-writes optional v1 step via callback """ def __init__( self, flow_dir: Path, *, flow_name: str = "", flow_id: str = "", surface: str = "windows", recording_mode: str = "auto", dump_fn: Optional[Callable[[], Any]] = None, write_v1_step: Optional[Callable[..., Any]] = None, stability_timeout_sec: float = 3.0, save_sensitive_screenshots: bool = False, defer_action_evidence: bool = False, on_action: Optional[Callable[[ActionRecord], None]] = None, on_action_update: Optional[Callable[[ActionRecord, str], None]] = None, ) -> None: self.flow_dir = Path(flow_dir) self.actions_dir = self.flow_dir / "actions" self.actions_dir.mkdir(parents=True, exist_ok=True) self.snapshots = SnapshotService( self.flow_dir, save_sensitive_screenshots=save_sensitive_screenshots, ) self.catalog = ElementCatalogService(self.flow_dir, flow_id=flow_id or flow_name) self.aggregator = InputAggregator(idle_sec=8.0 if surface == "browser" else 0.8) self.dump_fn = dump_fn self.defer_action_evidence = bool(defer_action_evidence) self.on_action = on_action self.on_action_update = on_action_update self.write_v1_step = write_v1_step self.stability_timeout_sec = stability_timeout_sec self._lock = threading.RLock() self._seq = 0 self.status = RecorderState.IDLE self.last_action: Optional[ActionRecord] = None self.last_warning: str = "" self.action_flow = ActionFlow( flow_id=flow_id or self.flow_dir.name, flow_name=flow_name or self.flow_dir.name, recording_mode=recording_mode, surface=surface, metadata={"action_capability": dict(ACTION_CAPABILITY)}, ) self._debounce_until = 0.0 self._debounce_sec = 0.35 self._last_click_signature = "" self._last_click_monotonic = 0.0 self._actions_by_id: dict[str, ActionRecord] = {} self._last_fill: dict[str, tuple[str, float]] = {} self._last_text_target: dict[str, Any] = {} def _dump_for_context(self, context: Optional[dict[str, Any]] = None) -> Any: if not self.dump_fn: return None ctx = dict(context or {}) kwargs = { key: ctx.get(key) for key in ("page_id", "document_id", "frame_id") if ctx.get(key) } if kwargs: try: return self.dump_fn(**kwargs) except TypeError as exc: if "unexpected keyword" not in str(exc) and "positional" not in str(exc): raise return self.dump_fn() @staticmethod def _event_context(event: RawUserEvent) -> dict[str, Any]: meta = dict(event.meta or {}) return { "surface": event.surface or "", "page_id": meta.get("page_id") or "", "document_id": meta.get("document_id") or "", "frame_id": meta.get("frame_id") or "", "url": event.url or "", "frame_url": (event.frame or {}).get("url") or "", "frame_name": (event.frame or {}).get("name") or "", "source_ts_ms": meta.get("source_ts_ms"), "event_id": meta.get("event_id") or "", "sequence": meta.get("sequence"), "timestamp": event.timestamp, "input_event": meta.get("input_event") or "", "is_trusted": meta.get("is_trusted"), } @staticmethod def _captured_live_counts(target: dict[str, Any]) -> Optional[dict[str, int]]: """Use uniqueness measured synchronously in the event-origin document.""" counts: dict[str, int] = {} for strategy, field in ( ("css", "css_count"), ("stable_css", "stable_css_count"), ("href", "href_count"), ): value = (target or {}).get(field) if value is not None: counts[strategy] = int(value) return counts or None def _stable_capture( self, context: dict[str, Any], *, reason: str, timeout: float ): return self.snapshots.wait_until_stable( lambda: self._dump_for_context(context), timeout_sec=timeout, change_reason=reason, ) def _action_capture(self, context: dict[str, Any], *, reason: str): """Capture one post-action observation without blocking on two dumps. Browser user events can arrive faster than a full page dump. A second stability poll made the FIFO lag by several seconds and contaminated earlier actions with later input. The action stream keeps ordering; quiet/checkpoint capture remains available for deliberately stable UI. """ import time time.sleep(0.06) dump = self._dump_for_context(context) return self.snapshots.capture_from_dump( dump, change_reason=reason, force_persist=True ) @staticmethod def _enrich_weak_browser_target(target: dict[str, Any], before) -> dict[str, Any]: """Attach a semantic hint without discarding the physical hit target.""" out = dict(target or {}) role = str(out.get("role") or "").lower() if role not in {"", "path", "svg", "img", "span", "div"}: return out if str(out.get("name") or out.get("text") or "").strip(): return out scope_text = " ".join( [str(out.get("css") or "")] + [ " ".join( str((ancestor or {}).get(key) or "") for key in ("id", "class_name", "css") ) for ancestor in (out.get("parent_chain") or []) ] ).lower() form_hints = ( "comment", "editor", "input", "form", "reply", "message", "chat", "评论", "输入", "回复", "消息", ) if not any(token in scope_text for token in form_hints): return out action_names = { "submit", "send", "publish", "post", "reply", "comment", "提交", "发送", "发布", "回复", "评论", "送信", "投稿", "公開", "보내기", "게시", "enviar", "publicar", "envoyer", "publier", "senden", "veröffentlichen", } candidates = [] for row in list(getattr(before, "locatable", None) or []): row_role = str(row.get("role") or row.get("control_type") or "").lower() row_name = str(row.get("name") or row.get("value") or row.get("text") or "").strip() if row_role in {"button", "menuitem", "link"} and row_name.lower() in action_names: candidates.append(row) unique = { (str(row.get("role") or row.get("control_type") or ""), str(row.get("name") or row.get("value") or row.get("text") or "")) for row in candidates } if len(unique) == 1: semantic_role, semantic_name = next(iter(unique)) out["semantic_role"] = semantic_role out["semantic_name"] = semantic_name out["semantic_inference"] = "form_scope_unique_action" out["semantic_confidence"] = 0.72 return out def start(self, *, entry_dump: Optional[Any] = None) -> ActionRecord | None: with self._lock: self.status = RecorderState.RECORDING self.action_flow.status = "open" self.action_flow.created_at = _now_iso() dump = entry_dump if dump is None and self.dump_fn: try: dump = self._dump_for_context() except Exception as exc: self.last_warning = f"entry dump failed: {exc}" dump = None if dump is not None: st = self.snapshots.capture_from_dump( dump, change_reason="entry", force_persist=True ) self.action_flow.entry_state_id = st.id self.action_flow.final_state_id = st.id if st.id not in self.action_flow.states: self.action_flow.states.append(st.id) if self.snapshots.latest_observation: self.action_flow.observations.append( self.snapshots.latest_observation.id ) # Optional v1 step for entry checkpoint if self.write_v1_step: try: self.write_v1_step(dump, label_hint="entry", change_reason="entry") except Exception: pass self._persist_flow() return None def pause(self) -> None: self.status = RecorderState.PAUSED def resume(self) -> None: if self.status == RecorderState.PAUSED: self.status = RecorderState.RECORDING def stop(self) -> ActionFlow: with self._lock: for item in self.aggregator.flush(): self._commit_fill(item) self.status = RecorderState.STOPPED self.action_flow.status = "ended" self.action_flow.ended_at = _now_iso() if self.snapshots.latest: self.action_flow.final_state_id = self.snapshots.latest.id self._persist_flow() self.catalog.save() return self.action_flow def handle_raw_event(self, event: RawUserEvent) -> Optional[ActionRecord]: if self.status != RecorderState.RECORDING: return None if event.from_inspector: return None with self._lock: kind = event.kind if kind in {"mouse_down", "click"}: return self._handle_click(event) if kind in {"key", "input", "value_change"}: return self._handle_input(event) if kind == "navigation": return self._handle_navigation(event) if kind == "checkpoint": return self.record_checkpoint(meta=event.meta) return None def record_checkpoint(self, *, meta: Optional[dict[str, Any]] = None) -> Optional[ActionRecord]: with self._lock: before = self.snapshots.latest before_obs = self.snapshots.latest_observation dump = None if self.dump_fn: try: dump = self._dump_for_context(meta) except Exception as exc: return self._failed_action( "checkpoint", error=str(exc), before=before ) after = self.snapshots.capture_from_dump( dump, change_reason="checkpoint", force_persist=True ) return self._finalize_action( action_type="checkpoint", before=before, after=after, target={}, input_payload={}, raw_event={"meta": meta or {}}, confidence=0.9, before_observation_id=before_obs.id if before_obs else None, after_observation_id=( self.snapshots.latest_observation.id if self.snapshots.latest_observation else None ), ) def _handle_click(self, event: RawUserEvent) -> Optional[ActionRecord]: import time now = time.monotonic() context = self._event_context(event) target = dict(event.target or {}) click_identity = element_identity_key( target, surface=event.surface, page_url=str(context.get("url") or ""), frame_name=str(context.get("frame_name") or ""), ) signature = "|".join( (click_identity, str(event.button or ""), str(event.x), str(event.y)) ) if ( signature == self._last_click_signature and now - self._last_click_monotonic < self._debounce_sec ): return None self._last_click_signature = signature self._last_click_monotonic = now # Flush pending input before click for item in self.aggregator.flush(): self._commit_fill(item) before = self.snapshots.latest before_obs = self.snapshots.latest_observation if event.surface == "browser": target = self._enrich_weak_browser_target(target, before) element = None live_counts = self._captured_live_counts(target) if ( not self.defer_action_evidence and event.surface == "browser" and context.get("page_id") and target ): try: import kit_browser live_counts = kit_browser.validate_element_on_page( page_id=str(context.get("page_id") or ""), element=target, frame=event.frame or None, ) except Exception as exc: self.last_warning = f"live locator validation failed: {exc}" if target: try: element = self.catalog.upsert_from_snapshot( target, surface=event.surface, window_scope=target.get("window_scope") or {}, rows=(before.locatable if before else None), frame=event.frame or None, live_counts=live_counts, ) except Exception as exc: self.last_warning = f"catalog upsert failed: {exc}" after = before status = "ok" err = "" if self.dump_fn and not self.defer_action_evidence: try: after = self._action_capture(context, reason="after_click") except Exception as exc: status = "partial" err = str(exc) try: dump = self._dump_for_context(context) after = self.snapshots.capture_from_dump( dump, change_reason="after_click_partial", force_persist=True ) except Exception as exc2: status = "failed" err = f"{err}; {exc2}" act = self._finalize_action( action_type="click", before=before, after=after, target=target, target_element_id=element.id if element else None, input_payload={}, raw_event=to_dict(event) if hasattr(event, "__dataclass_fields__") else dict(event.__dict__), status=status, confidence=0.8 if status == "ok" else 0.4, evidence={"error": err} if err else {}, started_at=event.timestamp, before_observation_id=before_obs.id if before_obs else None, after_observation_id=( self.snapshots.latest_observation.id if self.snapshots.latest_observation else None ), ) if self.write_v1_step and after is not None: # Dual-write one v1 step per meaningful action (not heartbeat) try: # Reconstruct a minimal dump-like for v1 from after state paths is hard; # caller should pass dump through write_v1_step if needed. pass except Exception: pass return act def _handle_input(self, event: RawUserEvent) -> Optional[ActionRecord]: key = event.key or (event.meta or {}).get("key") or "" text = event.text or (event.meta or {}).get("text") or (event.meta or {}).get("value") or "" common = { "target": event.target, "input_type": str((event.meta or {}).get("input_type") or ""), "field_name": str((event.target or {}).get("name") or ""), "capture_context": self._event_context(event), "raw_event": to_dict(event), } source_ms = (event.meta or {}).get("source_ts_ms") event_clock = ( float(source_ms) / 1000.0 if source_ms is not None else time.monotonic() ) if event.kind in {"input", "value_change"}: items = self.aggregator.on_value_change(value=text, now=event_clock, **common) else: items = self.aggregator.on_key(key=key, text=text, now=event_clock, **common) items.extend(self.aggregator.maybe_flush_idle(now=event_clock)) last = None for item in items: if item.action_type == "press_key": last = self._commit_press_key(item) else: last = self._commit_fill(item) return last def _handle_navigation(self, event: RawUserEvent) -> Optional[ActionRecord]: before = self.snapshots.latest before_obs = self.snapshots.latest_observation context = self._event_context(event) after = before if self.dump_fn and not self.defer_action_evidence: try: after = self._action_capture(context, reason="navigation") except Exception: pass return self._finalize_action( action_type="navigation", before=before, after=after, target={}, input_payload={"url": event.url}, raw_event={"url": event.url, "meta": event.meta}, confidence=0.7, started_at=event.timestamp, before_observation_id=before_obs.id if before_obs else None, after_observation_id=(self.snapshots.latest_observation.id if self.snapshots.latest_observation else None), ) def _commit_fill(self, item) -> Optional[ActionRecord]: if ( item.target and (self.action_flow.surface or "") == "windows" and not self._is_oversized_windows_target(item.target) ): self._last_text_target = dict(item.target) page_url = str(item.capture_context.get("url") or "") identity = element_identity_key( item.target or {}, surface=self.action_flow.surface or "windows", page_url=page_url, frame_name=str(item.capture_context.get("frame_name") or ""), ) text_value = str((item.input_payload or {}).get("text") or "") previous = self._last_fill.get(identity) source_ms = item.capture_context.get("source_ts_ms") now_marker = float(source_ms) / 1000.0 if source_ms else time.time() if previous and previous[0] == text_value and now_marker - previous[1] <= 15.0: return None self._last_fill[identity] = (text_value, now_marker) before = self.snapshots.latest before_obs = self.snapshots.latest_observation after = before if self.dump_fn and not self.defer_action_evidence: try: after = self._action_capture( item.capture_context, reason="after_fill" ) except Exception: pass element = None frame = { "url": item.capture_context.get("frame_url") or "", "name": item.capture_context.get("frame_name") or "", } live_counts = self._captured_live_counts(item.target or {}) if ( not self.defer_action_evidence and item.capture_context.get("page_id") and item.target ): try: import kit_browser live_counts = kit_browser.validate_element_on_page( page_id=str(item.capture_context.get("page_id") or ""), element=item.target, frame=frame, ) except Exception as exc: self.last_warning = f"live fill locator validation failed: {exc}" if item.target: try: element = self.catalog.upsert_from_snapshot( item.target, surface=self.action_flow.surface or "windows", window_scope=item.target.get("window_scope") or {}, rows=(before.locatable if before else None), frame=frame, live_counts=live_counts, ) except Exception: pass return self._finalize_action( action_type="fill", before=before, after=after, target=item.target, target_element_id=element.id if element else None, input_payload=item.input_payload, sensitive=item.sensitive, confidence=0.75, raw_event=item.raw_event or { "url": item.capture_context.get("url") or "", "meta": dict(item.capture_context or {}), }, raw_event_ids=item.raw_event_ids, started_at=str(item.capture_context.get("timestamp") or _now_iso()), before_observation_id=before_obs.id if before_obs else None, after_observation_id=(self.snapshots.latest_observation.id if self.snapshots.latest_observation else None), ) def _commit_press_key(self, item) -> ActionRecord: target = dict(item.target or {}) if ( (self.action_flow.surface or "") == "windows" and self._is_oversized_windows_target(target) and self._last_text_target and self._same_windows_target_scope(target, self._last_text_target) ): target = dict(self._last_text_target) before = self.snapshots.latest before_obs = self.snapshots.latest_observation after = before if self.dump_fn and not self.defer_action_evidence: try: after = self._action_capture( item.capture_context, reason="after_key" ) except Exception: pass element = None frame = { "url": item.capture_context.get("frame_url") or "", "name": item.capture_context.get("frame_name") or "", } live_counts = self._captured_live_counts(target) if target: try: element = self.catalog.upsert_from_snapshot( target, surface=self.action_flow.surface or "windows", window_scope=target.get("window_scope") or {}, rows=(before.locatable if before else None), frame=frame, live_counts=live_counts, ) except Exception as exc: self.last_warning = f"key target catalog upsert failed: {exc}" return self._finalize_action( action_type="press_key", before=before, after=after, target=target, target_element_id=element.id if element else None, input_payload=item.input_payload, sensitive=item.sensitive, confidence=0.7, raw_event=item.raw_event or { "kind": "key", "surface": self.action_flow.surface or "", "key": item.key, "target": target, "frame": frame, "url": item.capture_context.get("url") or "", "meta": dict(item.capture_context or {}), }, raw_event_ids=item.raw_event_ids, started_at=str(item.capture_context.get("timestamp") or _now_iso()), before_observation_id=before_obs.id if before_obs else None, after_observation_id=(self.snapshots.latest_observation.id if self.snapshots.latest_observation else None), ) def _failed_action( self, action_type: str, *, error: str, before ) -> ActionRecord: return self._finalize_action( action_type=action_type, before=before, after=before, target={}, input_payload={}, status="failed", confidence=0.1, evidence={"error": error}, ) def _finalize_action( self, *, action_type: str, before, after, target: dict[str, Any], input_payload: dict[str, Any], target_element_id: Optional[str] = None, sensitive: bool = False, raw_event: Optional[dict[str, Any]] = None, raw_event_ids: Optional[list[str]] = None, status: str = "ok", confidence: float = 0.5, evidence: Optional[dict[str, Any]] = None, before_observation_id: Optional[str] = None, after_observation_id: Optional[str] = None, started_at: Optional[str] = None, ) -> ActionRecord: self._seq += 1 payload, sens2 = redact_input_payload(input_payload or {}) sensitive = sensitive or sens2 clean_target = sanitize_artifact(target or {}, sensitive_hint=sensitive) clean_raw_event = sanitize_artifact(raw_event or {}, sensitive_hint=sensitive) before_id = before.id if before else None after_id = after.id if after else before_id if self.defer_action_evidence and action_type != "checkpoint": # A slow eventual dump must not be presented as an event-time # after-state. The evidence coordinator attaches it explicitly. after_id = None after_observation_id = None if after and after.id not in self.action_flow.states: self.action_flow.states.append(after.id) if before and before.id not in self.action_flow.states: self.action_flow.states.append(before.id) observed = [] proposed = [] if before and after: b = self.snapshots.to_summary_dict(before) a = self.snapshots.to_summary_dict(after) changes = diff_states(b, a) observed = changes_to_dicts(changes) proposed = postconditions_to_dicts(propose_postconditions(changes)) # Attach locator uniqueness hint for UI match_count = None unique = None primary_expr = "" if target_element_id: for el in self.catalog.catalog.elements: if el.id == target_element_id and el.primary_locator: match_count = el.primary_locator.actual_count unique = el.primary_locator.unique primary_expr = el.primary_locator.expression break if target_element_id not in self.action_flow.elements: self.action_flow.elements.append(target_element_id) if raw_event_ids is None: event_id = str(((clean_raw_event or {}).get("meta") or {}).get("event_id") or "") raw_event_ids = [event_id] if event_id else [] act = ActionRecord( sequence=self._seq, action_type=action_type, started_at=started_at or _now_iso(), ended_at=_now_iso(), surface=self.action_flow.surface or "", target_element_id=target_element_id, target_snapshot=dict(clean_target or {}), input=payload, sensitive=sensitive, before_state_id=before_id, after_state_id=after_id, before_observation_id=before_observation_id, after_observation_id=after_observation_id, observed_changes=observed, proposed_postconditions=proposed, status=status, confidence=confidence, evidence={ **(evidence or {}), "locator_match_count": match_count, "locator_unique": unique, "primary_locator": primary_expr, }, raw_event=dict(clean_raw_event or {}), raw_event_ids=list(raw_event_ids or []), ) write_model_json(self.actions_dir / f"{act.id}.json", act) self._actions_by_id[act.id] = act self.action_flow.actions.append(act.id) for observation_id in (before_observation_id, after_observation_id): if observation_id and observation_id not in self.action_flow.observations: self.action_flow.observations.append(observation_id) if after_id: self.action_flow.final_state_id = after_id self.last_action = act self._persist_flow() self._notify_action_update(act, "captured") if self.on_action is not None and action_type != "checkpoint": try: self.on_action(act) except Exception as exc: self.last_warning = f"action evidence scheduling failed: {exc}" # Dual-write optional pointer fields into flow.json via side file is enough; # artifacts integration updates flow.json separately. return act def attach_eventual_evidence( self, action_ids: list[str], *, dump: Any, requested_monotonic: float, ) -> None: """Attach one coalesced full dump without rewriting causal after-state.""" with self._lock: lag_ms = max(0, int((time.monotonic() - requested_monotonic) * 1000)) state = self.snapshots.capture_from_dump( dump, change_reason="eventual_after_actions", force_persist=True ) observation = self.snapshots.latest_observation if state.id not in self.action_flow.states: self.action_flow.states.append(state.id) if observation and observation.id not in self.action_flow.observations: self.action_flow.observations.append(observation.id) self.action_flow.final_state_id = state.id if (self.action_flow.surface or "") == "windows": self._promote_windows_targets_from_dump( dump, action_ids=action_ids, evidence_lag_ms=lag_ms ) for action_id in action_ids: action = self._actions_by_id.get(action_id) if action is None: continue action.evidence.update( { "eventual_observation_id": observation.id if observation else "", "eventual_state_id": state.id, "evidence_association": "coalesced_eventual", "evidence_lag_ms": lag_ms, } ) write_model_json(self.actions_dir / f"{action.id}.json", action) self._notify_action_update(action, "evidence") self._persist_flow() @staticmethod def _is_oversized_windows_target(target: dict[str, Any]) -> bool: bounds = dict(target.get("bounds") or target.get("rect") or {}) area = int(bounds.get("width") or 0) * int(bounds.get("height") or 0) return str(target.get("control_type") or "") == "WindowControl" or area > 1_000_000 @staticmethod def _is_broad_windows_target(target: dict[str, Any]) -> bool: if ActionRecordingManager._is_oversized_windows_target(target): return True bounds = dict(target.get("bounds") or target.get("rect") or {}) area = int(bounds.get("width") or 0) * int(bounds.get("height") or 0) control_type = str(target.get("control_type") or "") name = str(target.get("name") or target.get("automation_id") or "").lower() layout_tokens = ( "search_result", "listview", "list_view", "watermark", "container", "layout", "clientbg", "panel", "pane", ) return ( area > 250_000 or ( control_type in {"TreeControl", "PaneControl", "GroupControl"} and any(token in name for token in layout_tokens) ) ) @staticmethod def _unique_windows_row_candidate( rows: list[dict[str, Any]], ) -> Optional[dict[str, Any]]: """Return a single visible result/list row when geometry is unreliable.""" row_tokens = ( "user_item", "result_item", "search_item", "contact_item", "list_item", "listitem", "result_row", "contact_row", ) candidates: dict[tuple[Any, ...], tuple[int, dict[str, Any]]] = {} for index, row in enumerate(rows): if row.get("is_offscreen") is True: continue control_type = str(row.get("control_type") or "") name = " ".join( str(row.get(key) or "") for key in ("name", "automation_id", "class_name") ).lower() if not ( any(token in name for token in row_tokens) or control_type in {"ListItemControl", "DataItemControl"} ): continue bounds = dict(row.get("bounds") or row.get("rect") or {}) width = int(bounds.get("width") or 0) height = int(bounds.get("height") or 0) if width < 8 or height < 8: continue key = ( int(bounds.get("left") or 0), int(bounds.get("top") or 0), int(bounds.get("right") or 0), int(bounds.get("bottom") or 0), str(row.get("name") or ""), str(row.get("value") or ""), ) candidates[key] = (index, dict(row)) if len(candidates) != 1: return None index, chosen = next(iter(candidates.values())) chosen["bounds"] = dict(chosen.get("bounds") or chosen.get("rect") or {}) chosen.pop("rect", None) # A row container often has no accessible value of its own. Prefer an # explicit row-name descendant over structural labels such as # search_avatar_label, and strip WeCom rich-text markup. row_depth = int(chosen.get("depth") or 0) structural = { "layout", "container", "controlui", "horizontal", "vertical", "user_item", "list_item", "result_item", } descendants: list[tuple[int, str, str, dict[str, Any]]] = [] preferred_tokens = ( "search_name", "groupbuddyname", "contact_name", "display_name", "result_name", "title_label", ) rejected_tokens = ("avatar", "layout", "container", "controlui") for child in rows[index + 1 :]: child_depth = int(child.get("depth") or 0) if child_depth <= row_depth: break raw_value = str(child.get("value") or "").strip() name = str(child.get("name") or "").strip() clean_value = html.unescape(re.sub(r"<[^>]+>", "", raw_value)).strip() lowered = name.lower() semantic = clean_value or ( name if name and not any(token in lowered for token in structural) else "" ) if not semantic or semantic == "[REDACTED]" or any( token in lowered for token in rejected_tokens ): continue rank = 0 if any(token in lowered for token in preferred_tokens) else 1 descendants.append((rank, semantic, name, child)) if descendants: _rank, semantic, name, child = min(descendants, key=lambda item: item[0]) chosen["semantic_name"] = semantic chosen["row_descendant"] = { "name": name, "value": str(child.get("value") or "").strip(), "control_type": child.get("control_type") or "", } if not str(chosen.get("value") or ""): chosen["value"] = semantic return chosen @staticmethod def _windows_hit_candidate( rows: list[dict[str, Any]], x: int, y: int ) -> Optional[dict[str, Any]]: interactive = { "ButtonControl", "EditControl", "TreeItemControl", "ListItemControl", "MenuItemControl", "CheckBoxControl", "RadioButtonControl", "ComboBoxControl", "TabItemControl", "HyperlinkControl", } semantic_tokens = ( "search", "searchedit", "edit", "input", "message", "send", "comment", "reply", "搜索", "输入", "发送", "消息", ) candidates: list[tuple[tuple[int, int, int, int], dict[str, Any]]] = [] for row in rows: bounds = dict(row.get("bounds") or row.get("rect") or {}) try: inside = ( int(bounds.get("left")) <= x <= int(bounds.get("right")) and int(bounds.get("top")) <= y <= int(bounds.get("bottom")) ) except (TypeError, ValueError): continue if not inside or row.get("is_offscreen") is True: continue width = max(0, int(bounds.get("width") or 0)) height = max(0, int(bounds.get("height") or 0)) area = width * height control_type = str(row.get("control_type") or "") # Hairlines, separators and layout-only artifacts are never useful # click targets even when they happen to be the smallest hit. if ( not area or width < 4 or height < 4 or control_type == "WindowControl" ): continue text = " ".join( str(row.get(key) or "") for key in ("name", "value", "automation_id", "class_name") ).lower() if any(token in text for token in ("separator", "seperator", "divider", "splitter")): continue # A named editor/result is stronger than a smaller generic list # item covering the same point. exact_editor = any( token in text for token in ("searchedit", "send_rich_edit", "compose_box", "search_box") ) and "_layout" not in text if exact_editor: priority = 0 elif any(token in text for token in semantic_tokens): priority = 1 elif control_type in interactive: priority = 2 else: priority = 3 layout_penalty = 1 if "layout" in text or "container" in text else 0 redacted = 1 if "[redacted]" in text else 0 candidates.append(((priority, layout_penalty, redacted, area), row)) if not candidates: return None chosen = dict(min(candidates, key=lambda item: item[0])[1]) bounds = dict(chosen.get("bounds") or chosen.get("rect") or {}) chosen["bounds"] = bounds chosen.pop("rect", None) chosen["capture_point"] = {"x": x, "y": y} return chosen @staticmethod def _windows_candidate_quality(target: dict[str, Any]) -> tuple[int, int, int]: bounds = dict(target.get("bounds") or target.get("rect") or {}) area = int(bounds.get("width") or 0) * int(bounds.get("height") or 0) control_type = str(target.get("control_type") or "") text = " ".join( str(target.get(key) or "") for key in ("name", "value", "automation_id", "class_name") ).lower() interactive = control_type in { "ButtonControl", "EditControl", "TreeItemControl", "ListItemControl", "MenuItemControl", "CheckBoxControl", "RadioButtonControl", "ComboBoxControl", "TabItemControl", "HyperlinkControl", } semantic = any( token in text for token in ("search", "edit", "input", "message", "send", "buddy", "result") ) exact_editor = any( token in text for token in ("searchedit", "send_rich_edit", "compose_box", "search_box") ) and "_layout" not in text return (2 if exact_editor else (1 if semantic else 0), 1 if interactive else 0, -area) @staticmethod def _clean_windows_rich_text(value: str) -> str: return html.unescape(re.sub(r"<[^>]+>", "", str(value or ""))).strip() @staticmethod def _same_windows_target_scope( first: dict[str, Any], second: dict[str, Any] ) -> bool: a = dict(first.get("window_scope") or {}) b = dict(second.get("window_scope") or {}) ah, bh = int(a.get("hwnd") or 0), int(b.get("hwnd") or 0) if ah and bh: return ah == bh ap, bp = int(a.get("pid") or 0), int(b.get("pid") or 0) ac = str(a.get("class_name") or first.get("class_name") or "") bc = str(b.get("class_name") or second.get("class_name") or "") return bool(ap and ap == bp and (not ac or not bc or ac == bc)) @staticmethod def _windows_scope_key(target: dict[str, Any]) -> tuple[Any, ...]: scope = dict(target.get("window_scope") or {}) hwnd = int(scope.get("hwnd") or 0) if hwnd: return ("hwnd", hwnd) return ( "process", int(scope.get("pid") or 0), str(scope.get("class_name") or target.get("class_name") or ""), ) @staticmethod def _is_dynamic_windows_candidate(target: dict[str, Any]) -> bool: name = str(target.get("name") or target.get("automation_id") or "").lower() value = str(target.get("value") or "").strip() dynamic_tokens = ( "description", "preview", "snippet", "latest_message", "last_message", "timestamp", "time_label", "消息摘要", "最新消息", ) return bool(value and any(token in name for token in dynamic_tokens)) @staticmethod def _same_windows_scope(target: dict[str, Any], window: dict[str, Any]) -> bool: scope = dict(target.get("window_scope") or {}) target_hwnd = int(scope.get("hwnd") or 0) dump_hwnd = int(window.get("hwnd") or 0) if target_hwnd and dump_hwnd: return target_hwnd == dump_hwnd target_pid = int(scope.get("pid") or 0) dump_pid = int(window.get("pid") or 0) target_class = str(scope.get("class_name") or target.get("class_name") or "") dump_class = str(window.get("class_name") or "") if not scope: # Synthetic/unit captures may only carry the root class. Real # Windows events include hwnd/pid; class equality is the safest # compatibility fallback when that identity is unavailable. return bool(target_class and target_class == dump_class) return bool(target_pid and target_pid == dump_pid and target_class == dump_class) def _promote_windows_targets_from_dump( self, dump: Any, *, action_ids: Optional[list[str]] = None, evidence_lag_ms: int = 0, ) -> None: rows = list(getattr(dump, "locatable", None) or []) window = dict(getattr(dump, "window", None) or {}) if not rows: return last_business_by_scope: dict[tuple[Any, ...], dict[str, Any]] = {} allowed = set(action_ids or []) actions = sorted(self._actions_by_id.values(), key=lambda item: item.sequence) for action in actions: target = dict(action.target_snapshot or {}) scope_key = self._windows_scope_key(target) should_update = not allowed or action.id in allowed if not should_update: if target and not self._is_broad_windows_target(target): last_business_by_scope[scope_key] = target continue # Eventual evidence from another application or from WeCom's main # window must never rewrite a transient popup/screenshot action. if not self._same_windows_scope(target, window): continue promoted: Optional[dict[str, Any]] = None promotion_reason = "eventual_tree_coordinate_hit" if action.action_type == "click" and self._is_broad_windows_target(target): raw = dict(action.raw_event or {}) meta = dict(raw.get("meta") or {}) point = dict(meta.get("uia_coordinates") or {}) points: list[tuple[int, int]] = [] x = point.get("x", raw.get("x")) y = point.get("y", raw.get("y")) if x is not None and y is not None: points.append((int(x), int(y))) for candidate in meta.get("uia_coordinate_candidates") or []: cx, cy = candidate.get("x"), candidate.get("y") if cx is not None and cy is not None: p = (int(cx), int(cy)) if p not in points: points.append(p) hits = [ hit for px, py in points for hit in [self._windows_hit_candidate(rows, px, py)] if hit is not None ] if hits: promoted = max(hits, key=self._windows_candidate_quality) window_class = str(window.get("class_name") or "") if ( (not promoted or self._is_broad_windows_target(promoted)) and window_class == "SearchResultWindow2" ): unique_row = self._unique_windows_row_candidate(rows) if unique_row: promoted = unique_row promotion_reason = "eventual_tree_unique_row" elif action.action_type in {"fill", "type", "press_key"} and self._is_oversized_windows_target(target): promoted = dict(last_business_by_scope.get(scope_key) or {}) or None elif target and not self._is_broad_windows_target(target): last_business_by_scope[scope_key] = target if not promoted: continue if evidence_lag_ms > 5_000 and self._is_dynamic_windows_candidate(promoted): action.evidence.update( { "target_promotion_rejected": "late_dynamic_candidate", "rejected_candidate_name": promoted.get("name") or "", "rejected_evidence_lag_ms": int(evidence_lag_ms), } ) write_model_json(self.actions_dir / f"{action.id}.json", action) continue promoted["window_scope"] = dict( promoted.get("window_scope") or target.get("window_scope") or window ) promoted_name = str(promoted.get("name") or "").lower() if any( token in promoted_name for token in ("search_name", "contact_name", "groupbuddyname", "display_name") ): cleaned_value = self._clean_windows_rich_text( str(promoted.get("value") or "") ) if cleaned_value: promoted["value"] = cleaned_value promoted["semantic_name"] = cleaned_value element = self.catalog.upsert_from_snapshot( promoted, surface="windows", window_scope=promoted.get("window_scope") or {}, rows=rows, ) action.target_snapshot = sanitize_artifact(promoted) action.target_element_id = element.id action.confidence = max(float(action.confidence or 0.0), 0.72) action.evidence.update( { "target_promotion": promotion_reason, "original_target_control_type": target.get("control_type") or "", "locator_match_count": ( element.primary_locator.actual_count if element.primary_locator else None ), "locator_unique": ( element.primary_locator.unique if element.primary_locator else None ), "primary_locator": ( element.primary_locator.expression if element.primary_locator else "" ), } ) if isinstance(action.raw_event, dict): action.raw_event["target"] = sanitize_artifact(promoted) if element.id not in self.action_flow.elements: self.action_flow.elements.append(element.id) last_business_by_scope[scope_key] = promoted write_model_json(self.actions_dir / f"{action.id}.json", action) def note_evidence_failure(self, action_ids: list[str], error: str) -> None: with self._lock: self.last_warning = f"eventual evidence failed: {error}" for action_id in action_ids: action = self._actions_by_id.get(action_id) if action is None: continue action.evidence["eventual_evidence_error"] = str(error) write_model_json(self.actions_dir / f"{action.id}.json", action) self._notify_action_update(action, "evidence_failed") def _notify_action_update(self, action: ActionRecord, phase: str) -> None: if self.on_action_update is None: return try: self.on_action_update(action, phase) except Exception as exc: self.last_warning = f"action UI update failed: {exc}" def _persist_flow(self) -> None: write_model_json(self.flow_dir / "action_flow.json", self.action_flow) def heartbeat(self) -> None: """Lightweight monitor: flush idle input; do NOT create states/actions.""" if self.status != RecorderState.RECORDING: return with self._lock: for item in self.aggregator.maybe_flush_idle(): if item.action_type == "press_key": self._commit_press_key(item) else: self._commit_fill(item) def stats(self) -> dict[str, Any]: la = self.last_action return { "recorder_status": self.status, "state_count": len(self.action_flow.states), "observation_count": len(self.snapshots.observation_ids()), "action_count": len(self.action_flow.actions), "element_count": len(self.catalog.catalog.elements), "last_action_type": la.action_type if la else "", "last_target": (la.target_snapshot or {}).get("name") or (la.target_snapshot or {}).get("automation_id") or (la.target_snapshot or {}).get("semantic_name") or (la.target_snapshot or {}).get("class_name") or (la.target_snapshot or {}).get("tag") if la else "", "last_match_count": (la.evidence or {}).get("locator_match_count") if la else None, "last_unique": (la.evidence or {}).get("locator_unique") if la else None, "last_page_id": ((la.raw_event or {}).get("meta") or {}).get("page_id") if la else "", "last_observation_id": ( la.after_observation_id or (la.evidence or {}).get("eventual_observation_id") if la else "" ), "warning": self.last_warning, "recording_mode": self.action_flow.recording_mode, }