"""Event-driven Windows action recorder. Input is observed out-of-process with low-level Win32 hooks. Hook callbacks do only constant-time queue writes; UIA resolution happens on a separate worker so a slow provider (WeCom/SAP/etc.) cannot make Windows remove the hook. The old ``GetAsyncKeyState`` polling path remains only as a mouse fallback when hook installation is unavailable. """ from __future__ import annotations import ctypes import os import queue import threading import time from ctypes import wintypes from typing import Any, Callable, Optional from .base import RawUserEvent, RecorderBase, RecorderState # Win32 virtual key codes _VK_LBUTTON = 0x01 _VK_RBUTTON = 0x02 _VK_MBUTTON = 0x04 _WH_KEYBOARD_LL = 13 _WH_MOUSE_LL = 14 _WM_QUIT = 0x0012 _WM_KEYDOWN = 0x0100 _WM_SYSKEYDOWN = 0x0104 _WM_LBUTTONDOWN = 0x0201 _WM_LBUTTONUP = 0x0202 _WM_RBUTTONDOWN = 0x0204 _WM_RBUTTONUP = 0x0205 _LLMHF_INJECTED = 0x00000001 class _POINT(ctypes.Structure): _fields_ = [("x", wintypes.LONG), ("y", wintypes.LONG)] class _MSLLHOOKSTRUCT(ctypes.Structure): _fields_ = [ ("pt", _POINT), ("mouseData", wintypes.DWORD), ("flags", wintypes.DWORD), ("time", wintypes.DWORD), ("dwExtraInfo", ctypes.c_size_t), ] class _KBDLLHOOKSTRUCT(ctypes.Structure): _fields_ = [ ("vkCode", wintypes.DWORD), ("scanCode", wintypes.DWORD), ("flags", wintypes.DWORD), ("time", wintypes.DWORD), ("dwExtraInfo", ctypes.c_size_t), ] _SPECIAL_KEYS = { 0x08: "Backspace", 0x09: "Tab", 0x0D: "Enter", 0x1B: "Escape", 0x21: "PageUp", 0x22: "PageDown", 0x23: "End", 0x24: "Home", 0x25: "ArrowLeft", 0x26: "ArrowUp", 0x27: "ArrowRight", 0x28: "ArrowDown", 0x2E: "Delete", **{0x70 + i: f"F{i + 1}" for i in range(12)}, } class WindowsRecorder(RecorderBase): """Observe Windows input and resolve UIA targets outside target processes.""" capability_map = { "click": "implemented", "fill": "implemented", "press_key": "implemented", "double_click": "schema_only", "right_click": "partial", } def __init__( self, *, on_event: Optional[Callable[[RawUserEvent], None]] = None, exclude_hwnds: Optional[Callable[[], set[int]]] = None, poll_sec: float = 0.03, focus_value_poll_sec: float = 0.4, ) -> None: super().__init__(on_event=on_event, exclude_hwnds=exclude_hwnds) self.poll_sec = poll_sec self.focus_value_poll_sec = focus_value_poll_sec self._prev_down = False self._last_focus_value = "" self._last_focus_fp = "" self._pending_target: dict[str, Any] = {} self._last_uia_point: Optional[tuple[int, int]] = None self._last_raw_point: Optional[tuple[int, int]] = None self._last_root_hwnd = 0 self._last_actionable_target: dict[str, Any] = {} # Text editors need their own sticky target. A generic "last # actionable" target is unsafe here: selecting a contact immediately # before typing would otherwise make the contact row the input target. self._last_text_entry_target: dict[str, Any] = {} self._live_text_control: Any = None self._element_atlas: dict[str, list[dict[str, Any]]] = {} self._atlas_lock = threading.RLock() self._ime_composing = False self._last_typing_activity_identity = "" self._last_typing_activity_at = 0.0 self._input_queue: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=1024) self._capture_thread: Optional[threading.Thread] = None self._uia_thread: Optional[threading.Thread] = None self._uia_request = threading.Event() self._uia_finished = threading.Event() self._snapshot_lock = threading.RLock() self._latest_focus_snapshot: dict[str, Any] = {} self._latest_focus_sampled_at = 0.0 self._text_target_generation = 0 self._uia_bound_target_generation = -1 self._typing_active = False self._last_key_at = 0.0 self._accept_input = True self._startup_ready = threading.Event() self._hook_thread_id = 0 self._mouse_hook = None self._keyboard_hook = None self._mouse_proc = None self._keyboard_proc = None self._health_lock = threading.Lock() self._health: dict[str, Any] = { "backend": "starting", "mouse_hook": False, "keyboard_hook": False, "raw_input_events": 0, "mouse_events": 0, "keyboard_events": 0, "resolved_events": 0, "queue_dropped": 0, "loop_exceptions": 0, "last_error": "", "last_event_monotonic": None, "undrained_events": 0, "shutdown_complete": True, "uia_samples": 0, } @property def capability(self) -> dict[str, str]: return dict(self.capability_map) def start(self) -> None: if self._thread and self._thread.is_alive(): self.status = RecorderState.RECORDING return self._stop.clear() self._startup_ready.clear() self._uia_finished.clear() self._uia_request.clear() self._accept_input = True self._health_set(shutdown_complete=False, undrained_events=0) self.status = RecorderState.RECORDING self._capture_thread = threading.Thread( target=self._capture_loop, name="ui-inspector-win-input-consumer", daemon=True, ) self._capture_thread.start() self._uia_thread = threading.Thread( target=self._uia_sampler_loop, name="ui-inspector-win-uia-sampler", daemon=True, ) self._uia_thread.start() self._thread = threading.Thread( target=self._loop, name="ui-inspector-win-recorder", daemon=True ) self._thread.start() if not self._startup_ready.wait(timeout=2.0): self._record_error("Windows input backend did not complete startup handshake") def stop(self) -> None: # Stop accepting hook packets first. The consumer then drains every # packet already accepted while the independent UIA sampler performs # one final read. Never report a clean shutdown with a hidden tail. self._accept_input = False self._stop.set() self._uia_request.set() if self._hook_thread_id: try: ctypes.windll.user32.PostThreadMessageW( int(self._hook_thread_id), _WM_QUIT, 0, 0 ) except Exception as exc: self._record_error(exc) if self._thread and self._thread.is_alive(): self._thread.join(timeout=2.0) listener_alive = bool(self._thread and self._thread.is_alive()) if not listener_alive: self._thread = None if self._uia_thread and self._uia_thread.is_alive(): self._uia_thread.join(timeout=3.0) uia_alive = bool(self._uia_thread and self._uia_thread.is_alive()) if not uia_alive: self._uia_thread = None if self._capture_thread and self._capture_thread.is_alive(): self._capture_thread.join(timeout=5.0) resolver_alive = bool(self._capture_thread and self._capture_thread.is_alive()) if not resolver_alive: self._capture_thread = None undrained = int(self._input_queue.qsize()) complete = not listener_alive and not resolver_alive and undrained == 0 self._health_set( undrained_events=undrained, shutdown_complete=complete, uia_sampler_still_running=uia_alive, ) self.status = RecorderState.STOPPED if not complete: raise RuntimeError( "desktop recorder did not shut down cleanly: " f"undrained={undrained}, listener_alive={listener_alive}, " f"consumer_alive={resolver_alive}" ) def health(self) -> dict[str, Any]: with self._health_lock: result = dict(self._health) result["listener_thread_alive"] = bool( self._thread and self._thread.is_alive() ) result["resolver_thread_alive"] = bool( self._capture_thread and self._capture_thread.is_alive() ) result["uia_thread_alive"] = bool( self._uia_thread and self._uia_thread.is_alive() ) result["input_queue_size"] = self._input_queue.qsize() return result def _health_add(self, key: str, amount: int = 1) -> None: with self._health_lock: self._health[key] = int(self._health.get(key) or 0) + amount def _health_set(self, **values: Any) -> None: with self._health_lock: self._health.update(values) def _record_error(self, exc: Any) -> None: self._health_add("loop_exceptions") self._health_set(last_error=f"{type(exc).__name__}: {exc}" if isinstance(exc, BaseException) else str(exc)) def _loop(self) -> None: if os.name != "nt": self._health_set(backend="unsupported") self._startup_ready.set() return try: if self._run_hook_message_loop(): return except Exception as exc: self._record_error(exc) # Compatibility path: never the preferred backend. self._health_set(backend="get_async_key_state_fallback") self._startup_ready.set() self._poll_mouse_fallback() def _run_hook_message_loop(self) -> bool: user32 = ctypes.windll.user32 kernel32 = ctypes.windll.kernel32 lresult = ctypes.c_ssize_t hook_proc_type = ctypes.WINFUNCTYPE( lresult, ctypes.c_int, ctypes.c_size_t, ctypes.c_ssize_t ) user32.SetWindowsHookExW.argtypes = [ ctypes.c_int, hook_proc_type, ctypes.c_void_p, wintypes.DWORD ] user32.SetWindowsHookExW.restype = ctypes.c_void_p user32.CallNextHookEx.argtypes = [ ctypes.c_void_p, ctypes.c_int, ctypes.c_size_t, ctypes.c_ssize_t ] user32.CallNextHookEx.restype = lresult user32.UnhookWindowsHookEx.argtypes = [ctypes.c_void_p] user32.UnhookWindowsHookEx.restype = wintypes.BOOL kernel32.GetModuleHandleW.restype = ctypes.c_void_p @hook_proc_type def mouse_proc(code, wparam, lparam): if code >= 0 and wparam in { _WM_LBUTTONDOWN, _WM_LBUTTONUP, _WM_RBUTTONDOWN, _WM_RBUTTONUP }: data = ctypes.cast( lparam, ctypes.POINTER(_MSLLHOOKSTRUCT) ).contents self._enqueue_input( { "kind": "mouse", "message": int(wparam), "x": int(data.pt.x), "y": int(data.pt.y), "injected": bool(data.flags & _LLMHF_INJECTED), "source_time_ms": int(data.time), "coordinate_space": "physical_screen", } ) return user32.CallNextHookEx(None, code, wparam, lparam) @hook_proc_type def keyboard_proc(code, wparam, lparam): if code >= 0 and wparam in {_WM_KEYDOWN, _WM_SYSKEYDOWN}: data = ctypes.cast( lparam, ctypes.POINTER(_KBDLLHOOKSTRUCT) ).contents self._enqueue_input( { "kind": "key", "vk": int(data.vkCode), "scan": int(data.scanCode), "injected": bool(data.flags & 0x10), "source_time_ms": int(data.time), } ) return user32.CallNextHookEx(None, code, wparam, lparam) # Keep callbacks alive for the complete lifetime of the hooks. self._mouse_proc = mouse_proc self._keyboard_proc = keyboard_proc module = kernel32.GetModuleHandleW(None) self._hook_thread_id = int(kernel32.GetCurrentThreadId()) self._mouse_hook = user32.SetWindowsHookExW( _WH_MOUSE_LL, mouse_proc, module, 0 ) self._keyboard_hook = user32.SetWindowsHookExW( _WH_KEYBOARD_LL, keyboard_proc, module, 0 ) mouse_ok = bool(self._mouse_hook) keyboard_ok = bool(self._keyboard_hook) self._health_set( backend="low_level_hooks" if mouse_ok else "hook_failed", mouse_hook=mouse_ok, keyboard_hook=keyboard_ok, ) self._startup_ready.set() if not mouse_ok: if keyboard_ok: user32.UnhookWindowsHookEx(self._keyboard_hook) self._keyboard_hook = None return False msg = wintypes.MSG() try: while not self._stop.is_set(): result = int(user32.GetMessageW(ctypes.byref(msg), None, 0, 0)) if result <= 0: if result < 0: self._record_error("GetMessageW failed") break user32.TranslateMessage(ctypes.byref(msg)) user32.DispatchMessageW(ctypes.byref(msg)) finally: if self._mouse_hook: user32.UnhookWindowsHookEx(self._mouse_hook) if self._keyboard_hook: user32.UnhookWindowsHookEx(self._keyboard_hook) self._mouse_hook = None self._keyboard_hook = None self._hook_thread_id = 0 return True def _poll_mouse_fallback(self) -> None: user32 = ctypes.windll.user32 while not self._stop.is_set(): if self.status != RecorderState.RECORDING: time.sleep(0.05) continue try: down = bool(user32.GetAsyncKeyState(_VK_LBUTTON) & 0x8000) if down and not self._prev_down: x, y = self._cursor_pos(user32) self._enqueue_input( {"kind": "mouse", "message": _WM_LBUTTONDOWN, "x": x, "y": y, "fallback": True, "coordinate_space": "uia_screen"} ) elif not down and self._prev_down: self._enqueue_input({"kind": "mouse", "message": _WM_LBUTTONUP, "fallback": True}) self._prev_down = down except Exception as exc: self._record_error(exc) time.sleep(self.poll_sec) def _enqueue_input(self, packet: dict[str, Any]) -> None: if not self._accept_input: self._health_add("input_rejected_after_stop") return self._health_add("raw_input_events") self._health_add("mouse_events" if packet.get("kind") == "mouse" else "keyboard_events") self._health_set(last_event_monotonic=time.monotonic()) try: self._input_queue.put_nowait(packet) except queue.Full: self._health_add("queue_dropped") def _capture_loop(self) -> None: # This is deliberately a UIA-free path. Its only job is to preserve # input order and turn cached snapshots into semantic events quickly. while ( not self._stop.is_set() or not self._input_queue.empty() or not self._uia_finished.is_set() ): packet = None try: packet = self._input_queue.get(timeout=0.05) except queue.Empty: pass try: if packet is not None and self.status == RecorderState.RECORDING: self._dispatch_input_packet(packet) self._health_add("resolved_events") if ( self._typing_active and time.monotonic() - self._last_key_at >= 0.25 ): self._emit_cached_value_change(force=False) except Exception as exc: self._record_error(exc) finally: if packet is not None: self._input_queue.task_done() self._emit_cached_value_change(force=True) def _uia_sampler_loop(self) -> None: """Own all UIA/COM objects and publish plain-dict snapshots only.""" try: import desktop_uia as uia uia.ensure_com_sta() except Exception as exc: self._record_error(exc) try: while True: self._uia_request.wait(timeout=self.focus_value_poll_sec) self._uia_request.clear() try: with self._snapshot_lock: generation = self._text_target_generation requested_target = dict(self._last_text_entry_target or {}) if ( requested_target and generation != self._uia_bound_target_generation ): self._bind_text_target(requested_target, resolve_live=True) self._uia_bound_target_generation = generation # Once an editor has been bound, read it directly even if a # later click has moved OS focus. This preserves the final # committed value at transaction boundaries. live_info = self._snapshot_live_text_control() info = self._focused_control_snapshot() if live_info and self._typing_active: info = live_info if info: self._publish_focus_snapshot(info) self._health_add("uia_samples") except Exception as exc: self._record_error(exc) if self._stop.is_set(): break finally: self._uia_finished.set() def _publish_focus_snapshot(self, info: dict[str, Any]) -> None: """Cross the UIA thread boundary using data, never COM controls.""" if not info or self._is_transient_input_target(info): return snap = dict(info) with self._snapshot_lock: self._latest_focus_snapshot = snap self._latest_focus_sampled_at = time.monotonic() if self._is_text_entry(snap): self._last_text_entry_target = dict(snap) def _cached_focus_snapshot(self) -> dict[str, Any]: with self._snapshot_lock: return dict(self._latest_focus_snapshot or {}) @staticmethod def _focus_fingerprint(info: dict[str, Any]) -> str: scope = dict(info.get("window_scope") or {}) return "|".join( [ str(scope.get("hwnd") or 0), str(info.get("automation_id") or ""), str(info.get("name") or ""), str(info.get("control_type") or ""), ] ) def _emit_cached_value_change(self, *, force: bool) -> None: info = self._cached_focus_snapshot() if not info or not self._is_text_entry(info): return fp = self._focus_fingerprint(info) value = str(info.get("value") or "") if fp != self._last_focus_fp: # A late first sample may already contain the completed text. If # keys were observed, preserve it instead of mistaking it for the # baseline of a newly focused editor. if not (force and self._typing_active and value): self._last_focus_fp = fp self._last_focus_value = value return if value == self._last_focus_value: return self.emit( RawUserEvent( kind="value_change", surface="windows", text=value, target=info, meta={ "input_type": "password" if "password" in str(info.get("name") or "").lower() else "", "sampled_at": self._latest_focus_sampled_at, "causal_flush": bool(force), }, ) ) self._last_focus_fp = fp self._last_focus_value = value self._typing_active = False def _dispatch_input_packet(self, packet: dict[str, Any]) -> None: message = int(packet.get("message") or 0) if packet.get("kind") == "mouse": if message in {_WM_LBUTTONDOWN, _WM_RBUTTONDOWN}: self._on_mouse_down_at_fast( int(packet.get("x") or 0), int(packet.get("y") or 0), button="right" if message == _WM_RBUTTONDOWN else "left", source_meta=packet, ) elif message in {_WM_LBUTTONUP, _WM_RBUTTONUP}: self._on_mouse_up() elif packet.get("kind") == "key": self._on_key_down(packet) def _on_mouse_down(self, user32) -> None: point = self._cursor_pos(user32) self._on_mouse_down_at_fast(*point) def _on_mouse_down_at_fast( self, x: int, y: int, *, button: str = "left", source_meta: Optional[dict[str, Any]] = None, ) -> None: """Resolve a click without touching UIA. Win32 supplies application ownership and the persisted element atlas supplies an immediate semantic target. The sampler is then nudged to refine focus/value state asynchronously. """ self._emit_cached_value_change(force=True) raw_x, raw_y = int(x), int(y) user32 = ctypes.windll.user32 exclude: set[int] = set() try: exclude = set(self.exclude_hwnds() or set()) except Exception: pass hwnd = 0 root = 0 pid = 0 try: point = self._make_point(raw_x, raw_y) physical_lookup = getattr(user32, "WindowFromPhysicalPoint", None) if physical_lookup: physical_lookup.restype = ctypes.c_void_p hwnd = int(physical_lookup(point) or 0) else: user32.WindowFromPoint.restype = ctypes.c_void_p hwnd = int(user32.WindowFromPoint(point) or 0) root = int(user32.GetAncestor(hwnd, 2) or hwnd) if hwnd else 0 pid_ref = ctypes.c_ulong() if root: user32.GetWindowThreadProcessId(root, ctypes.byref(pid_ref)) pid = int(pid_ref.value or 0) if root in exclude or hwnd in exclude or pid == os.getpid(): return overlay = self._top_level_window_at_point( raw_x, raw_y, exclude_hwnds=exclude, preferred_pid=pid ) if overlay and overlay != root: root = int(overlay) pid_ref = ctypes.c_ulong() user32.GetWindowThreadProcessId(root, ctypes.byref(pid_ref)) pid = int(pid_ref.value or 0) except Exception as exc: self._record_error(exc) scope = self._window_scope_fast(root or hwnd, fallback_pid=pid) points = [(raw_x, raw_y)] scale = 1.0 try: get_dpi = getattr(user32, "GetDpiForWindow", None) dpi = int(get_dpi(int(root or hwnd)) or 96) if get_dpi and (root or hwnd) else 96 scale = max(1.0, dpi / 96.0) if scale > 1.05: points.append((round(raw_x / scale), round(raw_y / scale))) except Exception: pass target = self._atlas_hit(str(scope.get("class_name") or ""), points) if target: target["window_scope"] = dict(scope) selection = "persisted_element_atlas" else: target = { "name": str(scope.get("title") or ""), "control_type": "WindowControl", "class_name": str(scope.get("class_name") or ""), "window_scope": dict(scope), } selection = "win32_window_fallback" self._last_raw_point = (raw_x, raw_y) # Atlas coordinates are commonly logical on mixed-DPI desktop apps. self._last_uia_point = points[-1] self._last_root_hwnd = int(root or hwnd or 0) self._pending_target = dict(target) if not self._is_root_or_oversized(target): self._last_actionable_target = dict(target) if self._is_text_entry(target): self._set_text_target_fast(target) self.emit( RawUserEvent( kind="click", surface="windows", button=button, x=raw_x, y=raw_y, target=target, meta={ "input_backend": self.health().get("backend"), "injected": bool((source_meta or {}).get("injected")), "source_time_ms": (source_meta or {}).get("source_time_ms"), "source_coordinate_space": str( (source_meta or {}).get("coordinate_space") or "uia_screen" ), "raw_coordinates_physical": {"x": raw_x, "y": raw_y}, "coordinate_selection": selection, "dpi_scale": scale, "uia_deferred": True, }, ) ) self._uia_request.set() @staticmethod def _window_scope_fast(hwnd: int, *, fallback_pid: int = 0) -> dict[str, Any]: scope = {"hwnd": int(hwnd or 0), "pid": int(fallback_pid or 0), "title": "", "class_name": ""} if not hwnd: return scope try: user32 = ctypes.windll.user32 title = ctypes.create_unicode_buffer(512) klass = ctypes.create_unicode_buffer(256) user32.GetWindowTextW(int(hwnd), title, len(title)) user32.GetClassNameW(int(hwnd), klass, len(klass)) pid_ref = ctypes.c_ulong() user32.GetWindowThreadProcessId(int(hwnd), ctypes.byref(pid_ref)) scope.update( title=title.value, class_name=klass.value, pid=int(pid_ref.value or fallback_pid or 0), ) except Exception: pass return scope def _set_text_target_fast(self, info: dict[str, Any]) -> None: with self._snapshot_lock: self._last_text_entry_target = dict(info) self._text_target_generation += 1 self._last_focus_fp = self._focus_fingerprint(info) self._last_focus_value = str(info.get("value") or "") with self._snapshot_lock: # The atlas target is useful immediately. The sampler will replace # it with a live value-bearing snapshot when the provider responds. self._latest_focus_snapshot = dict(info) self._latest_focus_sampled_at = time.monotonic() self._typing_active = False self._uia_request.set() def _on_mouse_down_at( self, x: int, y: int, *, button: str = "left", source_meta: Optional[dict[str, Any]] = None, ) -> None: user32 = ctypes.windll.user32 # Preserve the final value of the previously focused editor before the # click changes focus (important for paste-then-click workflows). if self._last_focus_fp: self._poll_focus_value() raw_x, raw_y = int(x), int(y) exclude = set() try: exclude = set(self.exclude_hwnds() or set()) except Exception: exclude = set() # Filter Inspector window root = 0 try: point = self._make_point(raw_x, raw_y) physical_lookup = getattr(user32, "WindowFromPhysicalPoint", None) user32.WindowFromPoint.restype = ctypes.c_void_p if physical_lookup: physical_lookup.restype = ctypes.c_void_p hwnd = int( (physical_lookup(point) if physical_lookup else user32.WindowFromPoint(point)) or 0 ) root = int(user32.GetAncestor(hwnd, 2) or hwnd) if hwnd else 0 # GA_ROOT=2 target_pid = ctypes.c_ulong() if root: user32.GetWindowThreadProcessId(root, ctypes.byref(target_pid)) if ( root in exclude or hwnd in exclude or int(target_pid.value or 0) == os.getpid() ): self.emit( RawUserEvent( kind="mouse_down", surface="windows", x=x, y=y, from_inspector=True, meta={"reason": "inspector_hwnd"}, ) ) return overlay = self._top_level_window_at_point( raw_x, raw_y, exclude_hwnds=exclude, preferred_pid=int(target_pid.value or 0), ) if overlay and overlay != root: root = overlay target_pid = ctypes.c_ulong() user32.GetWindowThreadProcessId(root, ctypes.byref(target_pid)) except Exception: hwnd = 0 uia_x, uia_y, dpi_meta = self._uia_point_for_window( root or hwnd, raw_x, raw_y, coordinate_space=str((source_meta or {}).get("coordinate_space") or "uia_screen"), ) target, err = self._element_from_point(uia_x, uia_y) target_scope_hwnd = int((target.get("window_scope") or {}).get("hwnd") or 0) target_scope_root = ( int(user32.GetAncestor(target_scope_hwnd, 2) or target_scope_hwnd) if target_scope_hwnd else 0 ) if root and target_scope_root and target_scope_root != int(root): root_target, root_error = self._element_from_handle(int(root)) if root_target: target = root_target err = root_error target, uia_x, uia_y, probe_meta = self._probe_mixed_uia_coordinates( root or hwnd, raw_x, raw_y, current_point=(uia_x, uia_y), current_target=target or {}, ) dpi_meta.update(probe_meta) target_scope = dict((target or {}).get("window_scope") or {}) atlas_class = str(target_scope.get("class_name") or "") if self._is_root_or_oversized(target or {}) and atlas_class: atlas_points = [(int(uia_x), int(uia_y)), (raw_x, raw_y)] for candidate in probe_meta.get("uia_coordinate_candidates") or []: if candidate.get("x") is not None and candidate.get("y") is not None: atlas_points.append((int(candidate["x"]), int(candidate["y"]))) atlas_target = self._atlas_hit(atlas_class, atlas_points) if atlas_target: atlas_target["window_scope"] = dict(target_scope) target = atlas_target dpi_meta["coordinate_selection"] = "persisted_element_atlas" dpi_meta["uia_coordinates"] = {"x": uia_x, "y": uia_y} self._last_uia_point = (uia_x, uia_y) self._last_raw_point = (raw_x, raw_y) self._last_root_hwnd = int(root or hwnd or 0) self._pending_target = target or {} if self._pending_target and not self._is_root_or_oversized(self._pending_target): self._last_actionable_target = dict(self._pending_target) if self._is_text_entry(self._pending_target): self._bind_text_target(self._pending_target) self._seed_focus_baseline(self._pending_target) meta: dict[str, Any] = { "input_backend": self.health().get("backend"), "injected": bool((source_meta or {}).get("injected")), "source_time_ms": (source_meta or {}).get("source_time_ms"), **dpi_meta, } if err: meta["element_from_point_error"] = err meta["raw_coordinates"] = {"x": x, "y": y} # Coordinates saved as evidence only — never auto primary locator. meta["coordinates_are_not_primary_locator"] = True self.emit( RawUserEvent( kind="click", surface="windows", button=button, x=uia_x, y=uia_y, target=self._pending_target, meta=meta, ) ) @staticmethod def _uia_point_for_window( hwnd: int, x: int, y: int, *, coordinate_space: str, ) -> tuple[int, int, dict[str, Any]]: """Map WH_*_LL physical coordinates into the DPI-virtualized UIA space.""" meta: dict[str, Any] = { "raw_coordinates_physical": {"x": int(x), "y": int(y)}, "source_coordinate_space": coordinate_space, } if coordinate_space != "physical_screen" or not hwnd: meta["uia_coordinates"] = {"x": int(x), "y": int(y)} return int(x), int(y), meta try: # UIAutomation providers do not necessarily use the same coordinate # space as a DPI-aware WH_MOUSE_LL hook. In particular WeCom can # expose a 1440x900 UIA root while the hook and DWM both report a # 2880x1800 physical window. Comparing GetWindowRect with DWM alone # therefore incorrectly produces a scale of 1. Prefer the actual # UIA root rectangle as the destination coordinate space. physical = wintypes.RECT() hr = ctypes.windll.dwmapi.DwmGetWindowAttribute( int(hwnd), 9, ctypes.byref(physical), ctypes.sizeof(physical) ) if hr == 0: try: import uiautomation as auto root = auto.ControlFromHandle(int(hwnd)) rr = root.BoundingRectangle if root is not None else None rw = int(rr.right - rr.left) if rr is not None else 0 rh = int(rr.bottom - rr.top) if rr is not None else 0 pw = int(physical.right - physical.left) ph = int(physical.bottom - physical.top) if min(rw, rh, pw, ph) > 0: ux = round(int(rr.left) + (int(x) - physical.left) * rw / pw) uy = round(int(rr.top) + (int(y) - physical.top) * rh / ph) sx = pw / rw sy = ph / rh meta.update( { "uia_coordinates": {"x": ux, "y": uy}, "dpi_scale": {"x": round(sx, 4), "y": round(sy, 4)}, "coordinate_conversion": "dwm_physical_to_uia_root", } ) return ux, uy, meta except Exception as exc: meta["uia_root_coordinate_error"] = f"{type(exc).__name__}: {exc}" logical = wintypes.RECT() ctypes.windll.user32.GetWindowRect(int(hwnd), ctypes.byref(logical)) # Compatibility fallback for providers whose root rectangle cannot # be queried. DWMWA_EXTENDED_FRAME_BOUNDS is in physical pixels. lw = int(logical.right - logical.left) lh = int(logical.bottom - logical.top) pw = int(physical.right - physical.left) ph = int(physical.bottom - physical.top) if hr == 0 and min(lw, lh, pw, ph) > 0: sx = pw / lw sy = ph / lh ux = round(logical.left + (int(x) - physical.left) / sx) uy = round(logical.top + (int(y) - physical.top) / sy) meta.update( { "uia_coordinates": {"x": ux, "y": uy}, "dpi_scale": {"x": round(sx, 4), "y": round(sy, 4)}, "coordinate_conversion": "dwm_physical_to_getwindowrect_logical", } ) return ux, uy, meta except Exception as exc: meta["coordinate_conversion_error"] = f"{type(exc).__name__}: {exc}" meta["uia_coordinates"] = {"x": int(x), "y": int(y)} meta["coordinate_conversion"] = "identity_fallback" return int(x), int(y), meta @staticmethod def _target_quality(target: dict[str, Any]) -> tuple[int, int, int, int, int]: bounds = dict(target.get("bounds") or {}) width = max(0, int(bounds.get("width") or 0)) height = max(0, int(bounds.get("height") or 0)) area = width * height control_type = str(target.get("control_type") or "") text = " ".join( str(target.get(key) or "") for key in ("name", "automation_id", "class_name", "control_type") ).lower() oversized = control_type == "WindowControl" or area > 1_000_000 interactive = control_type in { "ButtonControl", "EditControl", "TreeItemControl", "ListItemControl", "MenuItemControl", "CheckBoxControl", "RadioButtonControl", "ComboBoxControl", "TabItemControl", "HyperlinkControl", } semantic = any( token in text for token in ( "search", "edit", "input", "message", "send", "contact", "buddy", "result", "comment", "reply", ) ) return ( 0 if oversized else 1, 1 if interactive else 0, 1 if semantic else 0, len(target.get("parent_chain") or []), -area, ) def _probe_mixed_uia_coordinates( self, hwnd: int, raw_x: int, raw_y: int, *, current_point: tuple[int, int], current_target: dict[str, Any], ) -> tuple[dict[str, Any], int, int, dict[str, Any]]: """Resolve providers whose root and descendants use different DPI spaces. WeCom exposes a physical-size root but logical-size descendants. Only probe alternative points when the direct hit is oversized and belongs to a known mixed-space window, so standard desktop applications retain their normal UIA hit-testing path. """ scope = dict(current_target.get("window_scope") or {}) window_class = str(scope.get("class_name") or current_target.get("class_name") or "") if ( not self._is_root_or_oversized(current_target) or window_class not in {"WeWorkWindow", "SearchResultWindow2"} ): return current_target, current_point[0], current_point[1], {} candidates: list[tuple[str, int, int, float]] = [ ("direct", int(current_point[0]), int(current_point[1]), 1.0) ] try: get_dpi = getattr(ctypes.windll.user32, "GetDpiForWindow", None) dpi = int(get_dpi(int(hwnd)) or 96) if get_dpi else 96 scale = max(1.0, float(dpi) / 96.0) if scale > 1.05: candidates.append( ( "window_dpi_absolute", round(int(raw_x) / scale), round(int(raw_y) / scale), scale, ) ) except Exception: pass unique: list[tuple[str, int, int, float]] = [] seen: set[tuple[int, int]] = set() for item in candidates: point = (item[1], item[2]) if point not in seen: seen.add(point) unique.append(item) expected_pid = int(scope.get("pid") or 0) expected_class = window_class best_target = current_target best_point = current_point best_label = "direct" best_scale = 1.0 best_quality = self._target_quality(current_target) attempts: list[dict[str, Any]] = [] for label, px, py, scale in unique[1:]: candidate, _err = self._element_from_point(px, py) candidate_scope = dict((candidate or {}).get("window_scope") or {}) same_scope = bool( candidate and (not expected_pid or int(candidate_scope.get("pid") or 0) == expected_pid) and ( not expected_class or str(candidate_scope.get("class_name") or "") == expected_class ) ) quality = self._target_quality(candidate or {}) if same_scope else (-1, 0, 0, 0, 0) attempts.append( { "strategy": label, "x": px, "y": py, "scale": round(scale, 4), "target": str((candidate or {}).get("name") or ""), "control_type": str((candidate or {}).get("control_type") or ""), "same_scope": same_scope, } ) if same_scope and quality > best_quality: best_target = candidate best_point = (px, py) best_label = label best_scale = scale best_quality = quality return ( best_target, int(best_point[0]), int(best_point[1]), { "uia_coordinate_candidates": attempts, "coordinate_selection": best_label, "mixed_coordinate_scale": round(best_scale, 4), }, ) def _on_key_down(self, packet: dict[str, Any]) -> None: key = _SPECIAL_KEYS.get(int(packet.get("vk") or 0), "") # Never call UIA here. Physical keys can arrive every few milliseconds; # this consumer must remain faster than the hook producer even when the # target accessibility provider is frozen for seconds. target = self._cached_focus_snapshot() if not self._is_text_entry(target): fallback = dict(self._last_text_entry_target or {}) if fallback and (not target or self._last_point_matches_target(fallback)): target = fallback if self._ime_composing: # Enter/Escape inside an IME candidate/composition window confirms # or cancels composition; it is not a business-level key action. # Keep the transaction attached to the last real editor. if target: self._typing_active = True self._last_key_at = time.monotonic() self._emit_typing_activity(target, packet) self._uia_request.set() return if not key: self._typing_active = True self._last_key_at = time.monotonic() self._emit_typing_activity(target, packet) self._uia_request.set() return if key in {"Backspace", "Delete"} and self._is_text_entry(target): # Editing keys are part of the resulting value transaction. The # next value poll records one complete fill instead of key spam. self._typing_active = True self._last_key_at = time.monotonic() self._emit_typing_activity(target, packet) self._uia_request.set() return # Enter/Tab/click are transaction boundaries: publish the latest # sampled value before the boundary action, preserving causal order. self._emit_cached_value_change(force=True) self.emit( RawUserEvent( kind="key", surface="windows", key=key, target=target, meta={ "input_backend": self.health().get("backend"), "injected": bool(packet.get("injected")), "source_time_ms": packet.get("source_time_ms"), }, ) ) self._uia_request.set() def _emit_typing_activity( self, target: dict[str, Any], packet: dict[str, Any] ) -> None: """Journal one marker per text transaction without persisting key text. The marker lets diagnostics distinguish "no typing happened" from "typing happened but the provider never exposed a value change". It is intentionally not reduced into an Action. """ target = dict(target or self._last_text_entry_target or {}) identity = "|".join( str(target.get(key) or "") for key in ("automation_id", "name", "control_type") ) scope = dict(target.get("window_scope") or {}) identity = f"{scope.get('hwnd') or 0}|{identity}" now = time.monotonic() if ( identity == self._last_typing_activity_identity and now - self._last_typing_activity_at < 0.8 ): self._last_typing_activity_at = now return self._last_typing_activity_identity = identity self._last_typing_activity_at = now self.emit( RawUserEvent( kind="typing_activity", surface="windows", target=target, meta={ "input_backend": self.health().get("backend"), "injected": bool(packet.get("injected")), "source_time_ms": packet.get("source_time_ms"), "content_recorded": False, "unresolved_target": not bool(target), "root_hwnd": int(self._last_root_hwnd or 0), "last_uia_point": ( {"x": self._last_uia_point[0], "y": self._last_uia_point[1]} if self._last_uia_point else {} ), }, ) ) def _focused_control_snapshot(self) -> dict[str, Any]: try: import desktop_uia as uia uia.ensure_com_sta() import uiautomation as auto ctrl = auto.GetFocusedControl() info = self._control_snapshot(ctrl) if ctrl is not None else {} self._ime_composing = self._is_transient_input_target(info) unusable = self._is_root_or_oversized(info) or self._is_transient_input_target(info) if unusable and self._last_uia_point: try: point_ctrl = auto.ControlFromPoint(*self._last_uia_point) point_info = self._control_snapshot( point_ctrl, x=self._last_uia_point[0], y=self._last_uia_point[1] ) if not self._is_root_or_oversized(point_info): info = point_info except Exception: pass if ( (self._is_root_or_oversized(info) or self._is_transient_input_target(info)) and self._last_raw_point and self._last_uia_point and self._last_root_hwnd ): try: probed, _x, _y, _meta = self._probe_mixed_uia_coordinates( self._last_root_hwnd, self._last_raw_point[0], self._last_raw_point[1], current_point=self._last_uia_point, current_target=info, ) if not self._is_root_or_oversized(probed) and not self._is_transient_input_target(probed): info = probed except Exception: pass if ( (self._is_root_or_oversized(info) or self._is_transient_input_target(info)) and self._last_text_entry_target ): refreshed = self._snapshot_live_text_control() if refreshed: info = refreshed if ( (self._is_root_or_oversized(info) or self._is_transient_input_target(info)) and self._last_actionable_target and self._last_point_matches_target(self._last_actionable_target) ): current_scope = dict(info.get("window_scope") or {}) last_scope = dict(self._last_actionable_target.get("window_scope") or {}) current_hwnd = int(current_scope.get("hwnd") or self._last_root_hwnd or 0) last_hwnd = int(last_scope.get("hwnd") or 0) if not current_hwnd or not last_hwnd or current_hwnd == last_hwnd: info = dict(self._last_actionable_target) if ( (self._is_root_or_oversized(info) or self._is_transient_input_target(info)) and self._last_uia_point ): scope = dict(info.get("window_scope") or {}) class_name = str(scope.get("class_name") or "") points = [self._last_uia_point] if self._last_raw_point: rx, ry = self._last_raw_point points.extend( [(rx, ry)] + [(round(rx / scale), round(ry / scale)) for scale in (1.25, 1.5, 1.75, 2.0, 2.5, 3.0)] ) atlas_target = self._atlas_hit(class_name, points) if atlas_target and self._is_text_entry(atlas_target): atlas_target["window_scope"] = dict(scope) self._bind_text_target(atlas_target) info = self._snapshot_live_text_control() or atlas_target if ( (self._is_root_or_oversized(info) or self._is_transient_input_target(info)) and self._last_text_entry_target ): current_scope = dict(info.get("window_scope") or {}) editor_scope = dict(self._last_text_entry_target.get("window_scope") or {}) current_pid = int(current_scope.get("pid") or 0) editor_pid = int(editor_scope.get("pid") or 0) # IME windows have a different HWND but normally share the # application's PID. Never let the IME HWND replace the # business editor just because the provider probe is cold. if not current_pid or not editor_pid or current_pid == editor_pid: refreshed = self._snapshot_live_text_control() info = refreshed or dict(self._last_text_entry_target) if info and not self._is_root_or_oversized(info) and not self._is_transient_input_target(info): self._last_actionable_target = dict(info) if self._is_text_entry(info): self._bind_text_target(info, resolve_live=False) return info # A transient OS surface must never cross the recorder boundary as # an application target. return {} except Exception as exc: self._record_error(exc) return {} def _on_mouse_up(self) -> None: # Click already emitted on down (capture before UI disappears). self._pending_target = {} def _poll_focus_value(self) -> None: try: info = self._focused_control_snapshot() if not info: return # Skip if focused control belongs to Inspector hwnd = int(info.get("window_scope", {}).get("hwnd") or 0) try: exclude = set(self.exclude_hwnds() or set()) except Exception: exclude = set() if hwnd and hwnd in exclude: return fp = "|".join( [ str(info.get("automation_id") or ""), str(info.get("name") or ""), str(info.get("control_type") or ""), ] ) val = str(info.get("value") or "") if fp == self._last_focus_fp and val == self._last_focus_value: return if self._last_focus_fp and fp == self._last_focus_fp and val != self._last_focus_value: self.emit( RawUserEvent( kind="value_change", surface="windows", text=val, target=info, meta={"input_type": "password" if "password" in (info.get("name") or "").lower() else ""}, ) ) self._last_focus_fp = fp self._last_focus_value = val info["fingerprint"] = fp except Exception: return @staticmethod def _is_text_entry(info: dict[str, Any]) -> bool: text = " ".join( str(info.get(key) or "") for key in ("name", "automation_id", "class_name", "control_type") ).lower() return str(info.get("control_type") or "") == "EditControl" or any( token in text for token in ("edit", "input", "search", "compose", "message", "rich") ) @staticmethod def _is_transient_input_target(info: dict[str, Any]) -> bool: text = " ".join( str(value or "") for value in ( info.get("name"), info.get("class_name"), (info.get("window_scope") or {}).get("class_name"), (info.get("window_scope") or {}).get("title"), ) ).lower() return any( token in text for token in ( "sopy_comp", "msctfime", "candidatewindow", "ime candidate", "inputmethod", "input method", "compositionwindow", "拼音", "候选词", "输入法", ) ) def _last_point_matches_target(self, target: dict[str, Any]) -> bool: """Guard stale focus fallback with the last click's geometry.""" bounds = dict(target.get("bounds") or {}) try: left = int(bounds.get("left")) top = int(bounds.get("top")) right = int(bounds.get("right")) bottom = int(bounds.get("bottom")) except (TypeError, ValueError): return False points: list[tuple[int, int]] = [] if self._last_uia_point: points.append(self._last_uia_point) if self._last_raw_point: rx, ry = self._last_raw_point points.append((rx, ry)) # Mixed-DPI providers commonly expose descendants at one of these # Windows scale factors even when their root uses physical pixels. for scale in (1.25, 1.5, 1.75, 2.0, 2.5, 3.0): points.append((round(rx / scale), round(ry / scale))) return any(left <= x <= right and top <= y <= bottom for x, y in points) def _seed_focus_baseline(self, info: dict[str, Any]) -> None: """Remember the pre-input value at click time. Without this seed, fast typing can finish before the first 400 ms focus poll and the completed text is mistaken for the initial baseline. """ if not info or not self._is_text_entry(info): return self._bind_text_target(info) self._last_focus_fp = "|".join( [ str(info.get("automation_id") or ""), str(info.get("name") or ""), str(info.get("control_type") or ""), ] ) self._last_focus_value = str(info.get("value") or "") def learn_elements( self, rows: list[dict[str, Any]], *, window: Optional[dict[str, Any]] = None, ) -> None: """Publish a bounded, value-free element atlas to the event-time path.""" win = dict(window or {}) class_name = str(win.get("class_name") or win.get("chosen_class") or "") if not class_name: return cleaned: list[dict[str, Any]] = [] for row in rows or []: item = dict(row or {}) bounds = dict(item.get("bounds") or item.get("rect") or {}) if not bounds: continue item["bounds"] = bounds item.pop("rect", None) # Historical atlases are locators, not a store of user content. item["value"] = "" item["window_scope"] = { "hwnd": int(win.get("hwnd") or 0), "pid": int(win.get("pid") or 0), "title": str(win.get("title") or ""), "class_name": class_name, } cleaned.append(item) if cleaned: with self._atlas_lock: self._element_atlas[class_name] = cleaned[-2500:] def _atlas_hit( self, class_name: str, points: list[tuple[int, int]] ) -> dict[str, Any]: with self._atlas_lock: rows = list(self._element_atlas.get(class_name) or []) candidates: list[tuple[tuple[int, int, int, int], dict[str, Any]]] = [] for row in rows: bounds = dict(row.get("bounds") or {}) try: hit = any( int(bounds.get("left")) <= x <= int(bounds.get("right")) and int(bounds.get("top")) <= y <= int(bounds.get("bottom")) for x, y in points ) except (TypeError, ValueError): continue if not hit or row.get("is_offscreen") is True: continue name = str(row.get("name") or row.get("automation_id") or "").lower() control_type = str(row.get("control_type") or "") width = max(0, int(bounds.get("width") or 0)) height = max(0, int(bounds.get("height") or 0)) area = width * height if not area or width < 4 or height < 4: continue exact_editor = name in { "searchedit", "send_rich_edit", "search_box", "compose_box" } editor = self._is_text_entry(row) and not name.endswith("_layout") semantic = any( token in name for token in ("search_name", "contact_name", "send", "edit", "input") ) candidates.append( ((1 if exact_editor else 0, 1 if editor else 0, 1 if semantic else 0, -area), row) ) return dict(max(candidates, default=((0, 0, 0, 0), {}), key=lambda x: x[0])[1]) def _bind_text_target( self, info: dict[str, Any], *, resolve_live: bool = True ) -> None: if not info or not self._is_text_entry(info): return self._last_text_entry_target = dict(info) if not resolve_live or not self._last_root_hwnd: return name = str(info.get("name") or "").strip() if not name: return try: import uiautomation as auto root = auto.ControlFromHandle(int(self._last_root_hwnd)) candidate = root.Control(searchDepth=24, Name=name) if candidate.Exists(0.35, 0.02): self._live_text_control = candidate except Exception: self._live_text_control = None def _snapshot_live_text_control(self) -> dict[str, Any]: if self._live_text_control is None: return {} try: info = self._control_snapshot(self._live_text_control) if info and self._is_text_entry(info): return info except Exception: pass self._live_text_control = None return {} @staticmethod def _is_root_or_oversized(info: dict[str, Any]) -> bool: bounds = dict(info.get("bounds") or {}) area = int(bounds.get("width") or 0) * int(bounds.get("height") or 0) return str(info.get("control_type") or "") == "WindowControl" or area > 1_000_000 def _element_from_point(self, x: int, y: int) -> tuple[dict[str, Any], str]: """Minimal directed read — NO ensure_ui_ready / full warm.""" try: import desktop_uia as uia uia.ensure_com_sta() import uiautomation as auto ctrl = auto.ControlFromPoint(int(x), int(y)) if ctrl is None: return {}, "ControlFromPoint returned None" return self._control_snapshot(ctrl, x=x, y=y), "" except Exception as exc: return {}, f"{type(exc).__name__}: {exc}" def _element_from_handle(self, hwnd: int) -> tuple[dict[str, Any], str]: try: import desktop_uia as uia uia.ensure_com_sta() import uiautomation as auto ctrl = auto.ControlFromHandle(int(hwnd)) if ctrl is None: return {}, "ControlFromHandle returned None" return self._control_snapshot(ctrl), "" except Exception as exc: return {}, f"{type(exc).__name__}: {exc}" @staticmethod def _is_decorative_window_class(class_name: str) -> bool: lowered = str(class_name or "").strip().lower() return lowered in { "shell_traywnd", "shell_secondarytraywnd", "progman", "workerw", "ime", "msctfime ui", "tooltips_class32", "sopy_comp", "perryshadowwnd", } or any( token in lowered for token in ("shadowwnd", "dropshadow", "drop_shadow", "windowshadow") ) @staticmethod def _top_level_window_at_point( x: int, y: int, *, exclude_hwnds: set[int], preferred_pid: int = 0, ) -> int: """Return the smallest visible top-level overlay containing a point. This is a generic Win32 fallback for applications whose accessibility provider reports the owner window from ``WindowFromPoint`` while an owned popup/list is visibly on top. """ if os.name != "nt": return 0 user32 = ctypes.windll.user32 candidates: list[tuple[int, int]] = [] @ctypes.WINFUNCTYPE(ctypes.c_bool, wintypes.HWND, wintypes.LPARAM) def callback(hwnd, _lparam): handle = int(hwnd or 0) if not handle or handle in exclude_hwnds or not user32.IsWindowVisible(handle): return True pid_ref = ctypes.c_ulong() user32.GetWindowThreadProcessId(handle, ctypes.byref(pid_ref)) pid = int(pid_ref.value or 0) if pid == os.getpid() or (preferred_pid and pid != preferred_pid): return True buf = ctypes.create_unicode_buffer(256) user32.GetClassNameW(handle, buf, len(buf)) class_name = buf.value if WindowsRecorder._is_decorative_window_class(class_name): return True rect = wintypes.RECT() if not user32.GetWindowRect(handle, ctypes.byref(rect)): return True if not (rect.left <= int(x) <= rect.right and rect.top <= int(y) <= rect.bottom): return True width = max(0, int(rect.right - rect.left)) height = max(0, int(rect.bottom - rect.top)) if width < 8 or height < 8: return True candidates.append((width * height, handle)) return True try: user32.EnumWindows(callback, 0) except Exception: return 0 return min(candidates, default=(0, 0))[1] def _control_snapshot( self, ctrl: Any, *, x: Optional[int] = None, y: Optional[int] = None ) -> dict[str, Any]: def safe(fn, default=""): try: return fn() except Exception: return default name = str(safe(lambda: ctrl.Name, "") or "") value = "" try: value = str(ctrl.GetValuePattern().Value) except Exception: value = str(safe(lambda: getattr(ctrl, "Value", ""), "") or "") if not value: try: value = str(ctrl.GetTextPattern().DocumentRange.GetText(-1) or "") except Exception: pass if not value: try: value = str(ctrl.GetLegacyIAccessiblePattern().Value or "") except Exception: pass aid = str(safe(lambda: ctrl.AutomationId, "") or "") ctype = str(safe(lambda: ctrl.ControlTypeName, "") or "") class_name = str(safe(lambda: ctrl.ClassName, "") or "") framework = str(safe(lambda: ctrl.FrameworkId, "") or "") enabled = safe(lambda: bool(ctrl.IsEnabled), None) offscreen = safe(lambda: bool(ctrl.IsOffscreen), None) rect: dict[str, int] = {} try: r = ctrl.BoundingRectangle rect = { "left": int(r.left), "top": int(r.top), "right": int(r.right), "bottom": int(r.bottom), "width": int(r.right - r.left), "height": int(r.bottom - r.top), } except Exception: rect = {} parent_chain: list[dict[str, str]] = [] try: p = ctrl.GetParentControl() depth = 0 while p is not None and depth < 8: parent_chain.append( { "name": str(getattr(p, "Name", "") or ""), "automation_id": str(getattr(p, "AutomationId", "") or ""), "control_type": str(getattr(p, "ControlTypeName", "") or ""), "class_name": str(getattr(p, "ClassName", "") or ""), } ) p = p.GetParentControl() depth += 1 except Exception: pass hwnd = 0 pid = 0 wtitle = "" wclass = "" try: hwnd = int(ctrl.NativeWindowHandle or 0) if not hwnd: # walk up for hwnd for item in parent_chain: del item p = ctrl for _ in range(12): h = int(getattr(p, "NativeWindowHandle", 0) or 0) if h: hwnd = h break p = p.GetParentControl() if p is None: break if hwnd: import desktop_uia as uia info = uia.window_info_from_hwnd(hwnd) if hasattr(uia, "window_info_from_hwnd") else None if info is None: # fallback via GetWindowText import ctypes user32 = ctypes.windll.user32 buf = ctypes.create_unicode_buffer(512) user32.GetWindowTextW(hwnd, buf, 512) wtitle = buf.value user32.GetClassNameW(hwnd, buf, 512) wclass = buf.value pid_c = ctypes.c_ulong() user32.GetWindowThreadProcessId(hwnd, ctypes.byref(pid_c)) pid = int(pid_c.value) else: wtitle = info.title wclass = info.class_name pid = info.pid hwnd = info.hwnd except Exception: pass snap = { "name": name, "value": value, "automation_id": aid, "control_type": ctype, "class_name": class_name, "framework_id": framework, "is_enabled": enabled, "is_offscreen": offscreen, "bounds": rect, "parent_chain": parent_chain, "window_scope": { "hwnd": hwnd, "pid": pid, "title": wtitle, "class_name": wclass, }, } if x is not None: snap["capture_point"] = {"x": x, "y": y} return snap @staticmethod def _cursor_pos(user32) -> tuple[int, int]: import ctypes from ctypes import wintypes class POINT(ctypes.Structure): _fields_ = [("x", wintypes.LONG), ("y", wintypes.LONG)] pt = POINT() user32.GetCursorPos(ctypes.byref(pt)) return int(pt.x), int(pt.y) @staticmethod def _make_point(x: int, y: int): import ctypes from ctypes import wintypes class POINT(ctypes.Structure): _fields_ = [("x", wintypes.LONG), ("y", wintypes.LONG)] return POINT(int(x), int(y))