chendelian
633ecff4e2
Keep the last in-template copy on main so history can restore it; product continues in jiangchang-ui-recorder. Co-authored-by: Cursor <cursoragent@cursor.com>
1712 lines
68 KiB
Python
1712 lines
68 KiB
Python
"""Event-driven Windows action recorder.
|
|
|
|
Input is observed out-of-process with low-level Win32 hooks. Hook callbacks do
|
|
only constant-time queue writes; UIA resolution happens on a separate worker so
|
|
a slow provider (WeCom/SAP/etc.) cannot make Windows remove the hook. The old
|
|
``GetAsyncKeyState`` polling path remains only as a mouse fallback when hook
|
|
installation is unavailable.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import ctypes
|
|
import os
|
|
import queue
|
|
import threading
|
|
import time
|
|
from ctypes import wintypes
|
|
from typing import Any, Callable, Optional
|
|
|
|
from .base import RawUserEvent, RecorderBase, RecorderState
|
|
|
|
# Win32 virtual key codes
|
|
_VK_LBUTTON = 0x01
|
|
_VK_RBUTTON = 0x02
|
|
_VK_MBUTTON = 0x04
|
|
|
|
_WH_KEYBOARD_LL = 13
|
|
_WH_MOUSE_LL = 14
|
|
_WM_QUIT = 0x0012
|
|
_WM_KEYDOWN = 0x0100
|
|
_WM_SYSKEYDOWN = 0x0104
|
|
_WM_LBUTTONDOWN = 0x0201
|
|
_WM_LBUTTONUP = 0x0202
|
|
_WM_RBUTTONDOWN = 0x0204
|
|
_WM_RBUTTONUP = 0x0205
|
|
_LLMHF_INJECTED = 0x00000001
|
|
|
|
|
|
class _POINT(ctypes.Structure):
|
|
_fields_ = [("x", wintypes.LONG), ("y", wintypes.LONG)]
|
|
|
|
|
|
class _MSLLHOOKSTRUCT(ctypes.Structure):
|
|
_fields_ = [
|
|
("pt", _POINT),
|
|
("mouseData", wintypes.DWORD),
|
|
("flags", wintypes.DWORD),
|
|
("time", wintypes.DWORD),
|
|
("dwExtraInfo", ctypes.c_size_t),
|
|
]
|
|
|
|
|
|
class _KBDLLHOOKSTRUCT(ctypes.Structure):
|
|
_fields_ = [
|
|
("vkCode", wintypes.DWORD),
|
|
("scanCode", wintypes.DWORD),
|
|
("flags", wintypes.DWORD),
|
|
("time", wintypes.DWORD),
|
|
("dwExtraInfo", ctypes.c_size_t),
|
|
]
|
|
|
|
|
|
_SPECIAL_KEYS = {
|
|
0x08: "Backspace",
|
|
0x09: "Tab",
|
|
0x0D: "Enter",
|
|
0x1B: "Escape",
|
|
0x21: "PageUp",
|
|
0x22: "PageDown",
|
|
0x23: "End",
|
|
0x24: "Home",
|
|
0x25: "ArrowLeft",
|
|
0x26: "ArrowUp",
|
|
0x27: "ArrowRight",
|
|
0x28: "ArrowDown",
|
|
0x2E: "Delete",
|
|
**{0x70 + i: f"F{i + 1}" for i in range(12)},
|
|
}
|
|
|
|
|
|
class WindowsRecorder(RecorderBase):
|
|
"""Observe Windows input and resolve UIA targets outside target processes."""
|
|
|
|
capability_map = {
|
|
"click": "implemented",
|
|
"fill": "implemented",
|
|
"press_key": "implemented",
|
|
"double_click": "schema_only",
|
|
"right_click": "partial",
|
|
}
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
on_event: Optional[Callable[[RawUserEvent], None]] = None,
|
|
exclude_hwnds: Optional[Callable[[], set[int]]] = None,
|
|
poll_sec: float = 0.03,
|
|
focus_value_poll_sec: float = 0.4,
|
|
) -> None:
|
|
super().__init__(on_event=on_event, exclude_hwnds=exclude_hwnds)
|
|
self.poll_sec = poll_sec
|
|
self.focus_value_poll_sec = focus_value_poll_sec
|
|
self._prev_down = False
|
|
self._last_focus_value = ""
|
|
self._last_focus_fp = ""
|
|
self._pending_target: dict[str, Any] = {}
|
|
self._last_uia_point: Optional[tuple[int, int]] = None
|
|
self._last_raw_point: Optional[tuple[int, int]] = None
|
|
self._last_root_hwnd = 0
|
|
self._last_actionable_target: dict[str, Any] = {}
|
|
# Text editors need their own sticky target. A generic "last
|
|
# actionable" target is unsafe here: selecting a contact immediately
|
|
# before typing would otherwise make the contact row the input target.
|
|
self._last_text_entry_target: dict[str, Any] = {}
|
|
self._live_text_control: Any = None
|
|
self._element_atlas: dict[str, list[dict[str, Any]]] = {}
|
|
self._atlas_lock = threading.RLock()
|
|
self._ime_composing = False
|
|
self._last_typing_activity_identity = ""
|
|
self._last_typing_activity_at = 0.0
|
|
self._input_queue: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=1024)
|
|
self._capture_thread: Optional[threading.Thread] = None
|
|
self._uia_thread: Optional[threading.Thread] = None
|
|
self._uia_request = threading.Event()
|
|
self._uia_finished = threading.Event()
|
|
self._snapshot_lock = threading.RLock()
|
|
self._latest_focus_snapshot: dict[str, Any] = {}
|
|
self._latest_focus_sampled_at = 0.0
|
|
self._text_target_generation = 0
|
|
self._uia_bound_target_generation = -1
|
|
self._typing_active = False
|
|
self._last_key_at = 0.0
|
|
self._accept_input = True
|
|
self._startup_ready = threading.Event()
|
|
self._hook_thread_id = 0
|
|
self._mouse_hook = None
|
|
self._keyboard_hook = None
|
|
self._mouse_proc = None
|
|
self._keyboard_proc = None
|
|
self._health_lock = threading.Lock()
|
|
self._health: dict[str, Any] = {
|
|
"backend": "starting",
|
|
"mouse_hook": False,
|
|
"keyboard_hook": False,
|
|
"raw_input_events": 0,
|
|
"mouse_events": 0,
|
|
"keyboard_events": 0,
|
|
"resolved_events": 0,
|
|
"queue_dropped": 0,
|
|
"loop_exceptions": 0,
|
|
"last_error": "",
|
|
"last_event_monotonic": None,
|
|
"undrained_events": 0,
|
|
"shutdown_complete": True,
|
|
"uia_samples": 0,
|
|
}
|
|
|
|
@property
|
|
def capability(self) -> dict[str, str]:
|
|
return dict(self.capability_map)
|
|
|
|
def start(self) -> None:
|
|
if self._thread and self._thread.is_alive():
|
|
self.status = RecorderState.RECORDING
|
|
return
|
|
self._stop.clear()
|
|
self._startup_ready.clear()
|
|
self._uia_finished.clear()
|
|
self._uia_request.clear()
|
|
self._accept_input = True
|
|
self._health_set(shutdown_complete=False, undrained_events=0)
|
|
self.status = RecorderState.RECORDING
|
|
self._capture_thread = threading.Thread(
|
|
target=self._capture_loop,
|
|
name="ui-inspector-win-input-consumer",
|
|
daemon=True,
|
|
)
|
|
self._capture_thread.start()
|
|
self._uia_thread = threading.Thread(
|
|
target=self._uia_sampler_loop,
|
|
name="ui-inspector-win-uia-sampler",
|
|
daemon=True,
|
|
)
|
|
self._uia_thread.start()
|
|
self._thread = threading.Thread(
|
|
target=self._loop, name="ui-inspector-win-recorder", daemon=True
|
|
)
|
|
self._thread.start()
|
|
if not self._startup_ready.wait(timeout=2.0):
|
|
self._record_error("Windows input backend did not complete startup handshake")
|
|
|
|
def stop(self) -> None:
|
|
# Stop accepting hook packets first. The consumer then drains every
|
|
# packet already accepted while the independent UIA sampler performs
|
|
# one final read. Never report a clean shutdown with a hidden tail.
|
|
self._accept_input = False
|
|
self._stop.set()
|
|
self._uia_request.set()
|
|
if self._hook_thread_id:
|
|
try:
|
|
ctypes.windll.user32.PostThreadMessageW(
|
|
int(self._hook_thread_id), _WM_QUIT, 0, 0
|
|
)
|
|
except Exception as exc:
|
|
self._record_error(exc)
|
|
if self._thread and self._thread.is_alive():
|
|
self._thread.join(timeout=2.0)
|
|
listener_alive = bool(self._thread and self._thread.is_alive())
|
|
if not listener_alive:
|
|
self._thread = None
|
|
if self._uia_thread and self._uia_thread.is_alive():
|
|
self._uia_thread.join(timeout=3.0)
|
|
uia_alive = bool(self._uia_thread and self._uia_thread.is_alive())
|
|
if not uia_alive:
|
|
self._uia_thread = None
|
|
if self._capture_thread and self._capture_thread.is_alive():
|
|
self._capture_thread.join(timeout=5.0)
|
|
resolver_alive = bool(self._capture_thread and self._capture_thread.is_alive())
|
|
if not resolver_alive:
|
|
self._capture_thread = None
|
|
undrained = int(self._input_queue.qsize())
|
|
complete = not listener_alive and not resolver_alive and undrained == 0
|
|
self._health_set(
|
|
undrained_events=undrained,
|
|
shutdown_complete=complete,
|
|
uia_sampler_still_running=uia_alive,
|
|
)
|
|
self.status = RecorderState.STOPPED
|
|
if not complete:
|
|
raise RuntimeError(
|
|
"desktop recorder did not shut down cleanly: "
|
|
f"undrained={undrained}, listener_alive={listener_alive}, "
|
|
f"consumer_alive={resolver_alive}"
|
|
)
|
|
|
|
def health(self) -> dict[str, Any]:
|
|
with self._health_lock:
|
|
result = dict(self._health)
|
|
result["listener_thread_alive"] = bool(
|
|
self._thread and self._thread.is_alive()
|
|
)
|
|
result["resolver_thread_alive"] = bool(
|
|
self._capture_thread and self._capture_thread.is_alive()
|
|
)
|
|
result["uia_thread_alive"] = bool(
|
|
self._uia_thread and self._uia_thread.is_alive()
|
|
)
|
|
result["input_queue_size"] = self._input_queue.qsize()
|
|
return result
|
|
|
|
def _health_add(self, key: str, amount: int = 1) -> None:
|
|
with self._health_lock:
|
|
self._health[key] = int(self._health.get(key) or 0) + amount
|
|
|
|
def _health_set(self, **values: Any) -> None:
|
|
with self._health_lock:
|
|
self._health.update(values)
|
|
|
|
def _record_error(self, exc: Any) -> None:
|
|
self._health_add("loop_exceptions")
|
|
self._health_set(last_error=f"{type(exc).__name__}: {exc}" if isinstance(exc, BaseException) else str(exc))
|
|
|
|
def _loop(self) -> None:
|
|
if os.name != "nt":
|
|
self._health_set(backend="unsupported")
|
|
self._startup_ready.set()
|
|
return
|
|
try:
|
|
if self._run_hook_message_loop():
|
|
return
|
|
except Exception as exc:
|
|
self._record_error(exc)
|
|
# Compatibility path: never the preferred backend.
|
|
self._health_set(backend="get_async_key_state_fallback")
|
|
self._startup_ready.set()
|
|
self._poll_mouse_fallback()
|
|
|
|
def _run_hook_message_loop(self) -> bool:
|
|
user32 = ctypes.windll.user32
|
|
kernel32 = ctypes.windll.kernel32
|
|
lresult = ctypes.c_ssize_t
|
|
hook_proc_type = ctypes.WINFUNCTYPE(
|
|
lresult, ctypes.c_int, ctypes.c_size_t, ctypes.c_ssize_t
|
|
)
|
|
user32.SetWindowsHookExW.argtypes = [
|
|
ctypes.c_int, hook_proc_type, ctypes.c_void_p, wintypes.DWORD
|
|
]
|
|
user32.SetWindowsHookExW.restype = ctypes.c_void_p
|
|
user32.CallNextHookEx.argtypes = [
|
|
ctypes.c_void_p, ctypes.c_int, ctypes.c_size_t, ctypes.c_ssize_t
|
|
]
|
|
user32.CallNextHookEx.restype = lresult
|
|
user32.UnhookWindowsHookEx.argtypes = [ctypes.c_void_p]
|
|
user32.UnhookWindowsHookEx.restype = wintypes.BOOL
|
|
kernel32.GetModuleHandleW.restype = ctypes.c_void_p
|
|
|
|
@hook_proc_type
|
|
def mouse_proc(code, wparam, lparam):
|
|
if code >= 0 and wparam in {
|
|
_WM_LBUTTONDOWN, _WM_LBUTTONUP, _WM_RBUTTONDOWN, _WM_RBUTTONUP
|
|
}:
|
|
data = ctypes.cast(
|
|
lparam, ctypes.POINTER(_MSLLHOOKSTRUCT)
|
|
).contents
|
|
self._enqueue_input(
|
|
{
|
|
"kind": "mouse",
|
|
"message": int(wparam),
|
|
"x": int(data.pt.x),
|
|
"y": int(data.pt.y),
|
|
"injected": bool(data.flags & _LLMHF_INJECTED),
|
|
"source_time_ms": int(data.time),
|
|
"coordinate_space": "physical_screen",
|
|
}
|
|
)
|
|
return user32.CallNextHookEx(None, code, wparam, lparam)
|
|
|
|
@hook_proc_type
|
|
def keyboard_proc(code, wparam, lparam):
|
|
if code >= 0 and wparam in {_WM_KEYDOWN, _WM_SYSKEYDOWN}:
|
|
data = ctypes.cast(
|
|
lparam, ctypes.POINTER(_KBDLLHOOKSTRUCT)
|
|
).contents
|
|
self._enqueue_input(
|
|
{
|
|
"kind": "key",
|
|
"vk": int(data.vkCode),
|
|
"scan": int(data.scanCode),
|
|
"injected": bool(data.flags & 0x10),
|
|
"source_time_ms": int(data.time),
|
|
}
|
|
)
|
|
return user32.CallNextHookEx(None, code, wparam, lparam)
|
|
|
|
# Keep callbacks alive for the complete lifetime of the hooks.
|
|
self._mouse_proc = mouse_proc
|
|
self._keyboard_proc = keyboard_proc
|
|
module = kernel32.GetModuleHandleW(None)
|
|
self._hook_thread_id = int(kernel32.GetCurrentThreadId())
|
|
self._mouse_hook = user32.SetWindowsHookExW(
|
|
_WH_MOUSE_LL, mouse_proc, module, 0
|
|
)
|
|
self._keyboard_hook = user32.SetWindowsHookExW(
|
|
_WH_KEYBOARD_LL, keyboard_proc, module, 0
|
|
)
|
|
mouse_ok = bool(self._mouse_hook)
|
|
keyboard_ok = bool(self._keyboard_hook)
|
|
self._health_set(
|
|
backend="low_level_hooks" if mouse_ok else "hook_failed",
|
|
mouse_hook=mouse_ok,
|
|
keyboard_hook=keyboard_ok,
|
|
)
|
|
self._startup_ready.set()
|
|
if not mouse_ok:
|
|
if keyboard_ok:
|
|
user32.UnhookWindowsHookEx(self._keyboard_hook)
|
|
self._keyboard_hook = None
|
|
return False
|
|
|
|
msg = wintypes.MSG()
|
|
try:
|
|
while not self._stop.is_set():
|
|
result = int(user32.GetMessageW(ctypes.byref(msg), None, 0, 0))
|
|
if result <= 0:
|
|
if result < 0:
|
|
self._record_error("GetMessageW failed")
|
|
break
|
|
user32.TranslateMessage(ctypes.byref(msg))
|
|
user32.DispatchMessageW(ctypes.byref(msg))
|
|
finally:
|
|
if self._mouse_hook:
|
|
user32.UnhookWindowsHookEx(self._mouse_hook)
|
|
if self._keyboard_hook:
|
|
user32.UnhookWindowsHookEx(self._keyboard_hook)
|
|
self._mouse_hook = None
|
|
self._keyboard_hook = None
|
|
self._hook_thread_id = 0
|
|
return True
|
|
|
|
def _poll_mouse_fallback(self) -> None:
|
|
user32 = ctypes.windll.user32
|
|
while not self._stop.is_set():
|
|
if self.status != RecorderState.RECORDING:
|
|
time.sleep(0.05)
|
|
continue
|
|
try:
|
|
down = bool(user32.GetAsyncKeyState(_VK_LBUTTON) & 0x8000)
|
|
if down and not self._prev_down:
|
|
x, y = self._cursor_pos(user32)
|
|
self._enqueue_input(
|
|
{"kind": "mouse", "message": _WM_LBUTTONDOWN, "x": x, "y": y, "fallback": True, "coordinate_space": "uia_screen"}
|
|
)
|
|
elif not down and self._prev_down:
|
|
self._enqueue_input({"kind": "mouse", "message": _WM_LBUTTONUP, "fallback": True})
|
|
self._prev_down = down
|
|
except Exception as exc:
|
|
self._record_error(exc)
|
|
time.sleep(self.poll_sec)
|
|
|
|
def _enqueue_input(self, packet: dict[str, Any]) -> None:
|
|
if not self._accept_input:
|
|
self._health_add("input_rejected_after_stop")
|
|
return
|
|
self._health_add("raw_input_events")
|
|
self._health_add("mouse_events" if packet.get("kind") == "mouse" else "keyboard_events")
|
|
self._health_set(last_event_monotonic=time.monotonic())
|
|
try:
|
|
self._input_queue.put_nowait(packet)
|
|
except queue.Full:
|
|
self._health_add("queue_dropped")
|
|
|
|
def _capture_loop(self) -> None:
|
|
# This is deliberately a UIA-free path. Its only job is to preserve
|
|
# input order and turn cached snapshots into semantic events quickly.
|
|
while (
|
|
not self._stop.is_set()
|
|
or not self._input_queue.empty()
|
|
or not self._uia_finished.is_set()
|
|
):
|
|
packet = None
|
|
try:
|
|
packet = self._input_queue.get(timeout=0.05)
|
|
except queue.Empty:
|
|
pass
|
|
try:
|
|
if packet is not None and self.status == RecorderState.RECORDING:
|
|
self._dispatch_input_packet(packet)
|
|
self._health_add("resolved_events")
|
|
if (
|
|
self._typing_active
|
|
and time.monotonic() - self._last_key_at >= 0.25
|
|
):
|
|
self._emit_cached_value_change(force=False)
|
|
except Exception as exc:
|
|
self._record_error(exc)
|
|
finally:
|
|
if packet is not None:
|
|
self._input_queue.task_done()
|
|
self._emit_cached_value_change(force=True)
|
|
|
|
def _uia_sampler_loop(self) -> None:
|
|
"""Own all UIA/COM objects and publish plain-dict snapshots only."""
|
|
try:
|
|
import desktop_uia as uia
|
|
uia.ensure_com_sta()
|
|
except Exception as exc:
|
|
self._record_error(exc)
|
|
try:
|
|
while True:
|
|
self._uia_request.wait(timeout=self.focus_value_poll_sec)
|
|
self._uia_request.clear()
|
|
try:
|
|
with self._snapshot_lock:
|
|
generation = self._text_target_generation
|
|
requested_target = dict(self._last_text_entry_target or {})
|
|
if (
|
|
requested_target
|
|
and generation != self._uia_bound_target_generation
|
|
):
|
|
self._bind_text_target(requested_target, resolve_live=True)
|
|
self._uia_bound_target_generation = generation
|
|
# Once an editor has been bound, read it directly even if a
|
|
# later click has moved OS focus. This preserves the final
|
|
# committed value at transaction boundaries.
|
|
live_info = self._snapshot_live_text_control()
|
|
info = self._focused_control_snapshot()
|
|
if live_info and self._typing_active:
|
|
info = live_info
|
|
if info:
|
|
self._publish_focus_snapshot(info)
|
|
self._health_add("uia_samples")
|
|
except Exception as exc:
|
|
self._record_error(exc)
|
|
if self._stop.is_set():
|
|
break
|
|
finally:
|
|
self._uia_finished.set()
|
|
|
|
def _publish_focus_snapshot(self, info: dict[str, Any]) -> None:
|
|
"""Cross the UIA thread boundary using data, never COM controls."""
|
|
if not info or self._is_transient_input_target(info):
|
|
return
|
|
snap = dict(info)
|
|
with self._snapshot_lock:
|
|
self._latest_focus_snapshot = snap
|
|
self._latest_focus_sampled_at = time.monotonic()
|
|
if self._is_text_entry(snap):
|
|
self._last_text_entry_target = dict(snap)
|
|
|
|
def _cached_focus_snapshot(self) -> dict[str, Any]:
|
|
with self._snapshot_lock:
|
|
return dict(self._latest_focus_snapshot or {})
|
|
|
|
@staticmethod
|
|
def _focus_fingerprint(info: dict[str, Any]) -> str:
|
|
scope = dict(info.get("window_scope") or {})
|
|
return "|".join(
|
|
[
|
|
str(scope.get("hwnd") or 0),
|
|
str(info.get("automation_id") or ""),
|
|
str(info.get("name") or ""),
|
|
str(info.get("control_type") or ""),
|
|
]
|
|
)
|
|
|
|
def _emit_cached_value_change(self, *, force: bool) -> None:
|
|
info = self._cached_focus_snapshot()
|
|
if not info or not self._is_text_entry(info):
|
|
return
|
|
fp = self._focus_fingerprint(info)
|
|
value = str(info.get("value") or "")
|
|
if fp != self._last_focus_fp:
|
|
# A late first sample may already contain the completed text. If
|
|
# keys were observed, preserve it instead of mistaking it for the
|
|
# baseline of a newly focused editor.
|
|
if not (force and self._typing_active and value):
|
|
self._last_focus_fp = fp
|
|
self._last_focus_value = value
|
|
return
|
|
if value == self._last_focus_value:
|
|
return
|
|
self.emit(
|
|
RawUserEvent(
|
|
kind="value_change",
|
|
surface="windows",
|
|
text=value,
|
|
target=info,
|
|
meta={
|
|
"input_type": "password"
|
|
if "password" in str(info.get("name") or "").lower()
|
|
else "",
|
|
"sampled_at": self._latest_focus_sampled_at,
|
|
"causal_flush": bool(force),
|
|
},
|
|
)
|
|
)
|
|
self._last_focus_fp = fp
|
|
self._last_focus_value = value
|
|
self._typing_active = False
|
|
|
|
def _dispatch_input_packet(self, packet: dict[str, Any]) -> None:
|
|
message = int(packet.get("message") or 0)
|
|
if packet.get("kind") == "mouse":
|
|
if message in {_WM_LBUTTONDOWN, _WM_RBUTTONDOWN}:
|
|
self._on_mouse_down_at_fast(
|
|
int(packet.get("x") or 0),
|
|
int(packet.get("y") or 0),
|
|
button="right" if message == _WM_RBUTTONDOWN else "left",
|
|
source_meta=packet,
|
|
)
|
|
elif message in {_WM_LBUTTONUP, _WM_RBUTTONUP}:
|
|
self._on_mouse_up()
|
|
elif packet.get("kind") == "key":
|
|
self._on_key_down(packet)
|
|
|
|
def _on_mouse_down(self, user32) -> None:
|
|
point = self._cursor_pos(user32)
|
|
self._on_mouse_down_at_fast(*point)
|
|
|
|
def _on_mouse_down_at_fast(
|
|
self,
|
|
x: int,
|
|
y: int,
|
|
*,
|
|
button: str = "left",
|
|
source_meta: Optional[dict[str, Any]] = None,
|
|
) -> None:
|
|
"""Resolve a click without touching UIA.
|
|
|
|
Win32 supplies application ownership and the persisted element atlas
|
|
supplies an immediate semantic target. The sampler is then nudged to
|
|
refine focus/value state asynchronously.
|
|
"""
|
|
self._emit_cached_value_change(force=True)
|
|
raw_x, raw_y = int(x), int(y)
|
|
user32 = ctypes.windll.user32
|
|
exclude: set[int] = set()
|
|
try:
|
|
exclude = set(self.exclude_hwnds() or set())
|
|
except Exception:
|
|
pass
|
|
hwnd = 0
|
|
root = 0
|
|
pid = 0
|
|
try:
|
|
point = self._make_point(raw_x, raw_y)
|
|
physical_lookup = getattr(user32, "WindowFromPhysicalPoint", None)
|
|
if physical_lookup:
|
|
physical_lookup.restype = ctypes.c_void_p
|
|
hwnd = int(physical_lookup(point) or 0)
|
|
else:
|
|
user32.WindowFromPoint.restype = ctypes.c_void_p
|
|
hwnd = int(user32.WindowFromPoint(point) or 0)
|
|
root = int(user32.GetAncestor(hwnd, 2) or hwnd) if hwnd else 0
|
|
pid_ref = ctypes.c_ulong()
|
|
if root:
|
|
user32.GetWindowThreadProcessId(root, ctypes.byref(pid_ref))
|
|
pid = int(pid_ref.value or 0)
|
|
if root in exclude or hwnd in exclude or pid == os.getpid():
|
|
return
|
|
overlay = self._top_level_window_at_point(
|
|
raw_x, raw_y, exclude_hwnds=exclude, preferred_pid=pid
|
|
)
|
|
if overlay and overlay != root:
|
|
root = int(overlay)
|
|
pid_ref = ctypes.c_ulong()
|
|
user32.GetWindowThreadProcessId(root, ctypes.byref(pid_ref))
|
|
pid = int(pid_ref.value or 0)
|
|
except Exception as exc:
|
|
self._record_error(exc)
|
|
|
|
scope = self._window_scope_fast(root or hwnd, fallback_pid=pid)
|
|
points = [(raw_x, raw_y)]
|
|
scale = 1.0
|
|
try:
|
|
get_dpi = getattr(user32, "GetDpiForWindow", None)
|
|
dpi = int(get_dpi(int(root or hwnd)) or 96) if get_dpi and (root or hwnd) else 96
|
|
scale = max(1.0, dpi / 96.0)
|
|
if scale > 1.05:
|
|
points.append((round(raw_x / scale), round(raw_y / scale)))
|
|
except Exception:
|
|
pass
|
|
target = self._atlas_hit(str(scope.get("class_name") or ""), points)
|
|
if target:
|
|
target["window_scope"] = dict(scope)
|
|
selection = "persisted_element_atlas"
|
|
else:
|
|
target = {
|
|
"name": str(scope.get("title") or ""),
|
|
"control_type": "WindowControl",
|
|
"class_name": str(scope.get("class_name") or ""),
|
|
"window_scope": dict(scope),
|
|
}
|
|
selection = "win32_window_fallback"
|
|
|
|
self._last_raw_point = (raw_x, raw_y)
|
|
# Atlas coordinates are commonly logical on mixed-DPI desktop apps.
|
|
self._last_uia_point = points[-1]
|
|
self._last_root_hwnd = int(root or hwnd or 0)
|
|
self._pending_target = dict(target)
|
|
if not self._is_root_or_oversized(target):
|
|
self._last_actionable_target = dict(target)
|
|
if self._is_text_entry(target):
|
|
self._set_text_target_fast(target)
|
|
self.emit(
|
|
RawUserEvent(
|
|
kind="click",
|
|
surface="windows",
|
|
button=button,
|
|
x=raw_x,
|
|
y=raw_y,
|
|
target=target,
|
|
meta={
|
|
"input_backend": self.health().get("backend"),
|
|
"injected": bool((source_meta or {}).get("injected")),
|
|
"source_time_ms": (source_meta or {}).get("source_time_ms"),
|
|
"source_coordinate_space": str(
|
|
(source_meta or {}).get("coordinate_space") or "uia_screen"
|
|
),
|
|
"raw_coordinates_physical": {"x": raw_x, "y": raw_y},
|
|
"coordinate_selection": selection,
|
|
"dpi_scale": scale,
|
|
"uia_deferred": True,
|
|
},
|
|
)
|
|
)
|
|
self._uia_request.set()
|
|
|
|
@staticmethod
|
|
def _window_scope_fast(hwnd: int, *, fallback_pid: int = 0) -> dict[str, Any]:
|
|
scope = {"hwnd": int(hwnd or 0), "pid": int(fallback_pid or 0), "title": "", "class_name": ""}
|
|
if not hwnd:
|
|
return scope
|
|
try:
|
|
user32 = ctypes.windll.user32
|
|
title = ctypes.create_unicode_buffer(512)
|
|
klass = ctypes.create_unicode_buffer(256)
|
|
user32.GetWindowTextW(int(hwnd), title, len(title))
|
|
user32.GetClassNameW(int(hwnd), klass, len(klass))
|
|
pid_ref = ctypes.c_ulong()
|
|
user32.GetWindowThreadProcessId(int(hwnd), ctypes.byref(pid_ref))
|
|
scope.update(
|
|
title=title.value,
|
|
class_name=klass.value,
|
|
pid=int(pid_ref.value or fallback_pid or 0),
|
|
)
|
|
except Exception:
|
|
pass
|
|
return scope
|
|
|
|
def _set_text_target_fast(self, info: dict[str, Any]) -> None:
|
|
with self._snapshot_lock:
|
|
self._last_text_entry_target = dict(info)
|
|
self._text_target_generation += 1
|
|
self._last_focus_fp = self._focus_fingerprint(info)
|
|
self._last_focus_value = str(info.get("value") or "")
|
|
with self._snapshot_lock:
|
|
# The atlas target is useful immediately. The sampler will replace
|
|
# it with a live value-bearing snapshot when the provider responds.
|
|
self._latest_focus_snapshot = dict(info)
|
|
self._latest_focus_sampled_at = time.monotonic()
|
|
self._typing_active = False
|
|
self._uia_request.set()
|
|
|
|
def _on_mouse_down_at(
|
|
self,
|
|
x: int,
|
|
y: int,
|
|
*,
|
|
button: str = "left",
|
|
source_meta: Optional[dict[str, Any]] = None,
|
|
) -> None:
|
|
user32 = ctypes.windll.user32
|
|
# Preserve the final value of the previously focused editor before the
|
|
# click changes focus (important for paste-then-click workflows).
|
|
if self._last_focus_fp:
|
|
self._poll_focus_value()
|
|
raw_x, raw_y = int(x), int(y)
|
|
exclude = set()
|
|
try:
|
|
exclude = set(self.exclude_hwnds() or set())
|
|
except Exception:
|
|
exclude = set()
|
|
|
|
# Filter Inspector window
|
|
root = 0
|
|
try:
|
|
point = self._make_point(raw_x, raw_y)
|
|
physical_lookup = getattr(user32, "WindowFromPhysicalPoint", None)
|
|
user32.WindowFromPoint.restype = ctypes.c_void_p
|
|
if physical_lookup:
|
|
physical_lookup.restype = ctypes.c_void_p
|
|
hwnd = int(
|
|
(physical_lookup(point) if physical_lookup else user32.WindowFromPoint(point))
|
|
or 0
|
|
)
|
|
root = int(user32.GetAncestor(hwnd, 2) or hwnd) if hwnd else 0 # GA_ROOT=2
|
|
target_pid = ctypes.c_ulong()
|
|
if root:
|
|
user32.GetWindowThreadProcessId(root, ctypes.byref(target_pid))
|
|
if (
|
|
root in exclude
|
|
or hwnd in exclude
|
|
or int(target_pid.value or 0) == os.getpid()
|
|
):
|
|
self.emit(
|
|
RawUserEvent(
|
|
kind="mouse_down",
|
|
surface="windows",
|
|
x=x,
|
|
y=y,
|
|
from_inspector=True,
|
|
meta={"reason": "inspector_hwnd"},
|
|
)
|
|
)
|
|
return
|
|
overlay = self._top_level_window_at_point(
|
|
raw_x,
|
|
raw_y,
|
|
exclude_hwnds=exclude,
|
|
preferred_pid=int(target_pid.value or 0),
|
|
)
|
|
if overlay and overlay != root:
|
|
root = overlay
|
|
target_pid = ctypes.c_ulong()
|
|
user32.GetWindowThreadProcessId(root, ctypes.byref(target_pid))
|
|
except Exception:
|
|
hwnd = 0
|
|
|
|
uia_x, uia_y, dpi_meta = self._uia_point_for_window(
|
|
root or hwnd, raw_x, raw_y,
|
|
coordinate_space=str((source_meta or {}).get("coordinate_space") or "uia_screen"),
|
|
)
|
|
target, err = self._element_from_point(uia_x, uia_y)
|
|
target_scope_hwnd = int((target.get("window_scope") or {}).get("hwnd") or 0)
|
|
target_scope_root = (
|
|
int(user32.GetAncestor(target_scope_hwnd, 2) or target_scope_hwnd)
|
|
if target_scope_hwnd
|
|
else 0
|
|
)
|
|
if root and target_scope_root and target_scope_root != int(root):
|
|
root_target, root_error = self._element_from_handle(int(root))
|
|
if root_target:
|
|
target = root_target
|
|
err = root_error
|
|
target, uia_x, uia_y, probe_meta = self._probe_mixed_uia_coordinates(
|
|
root or hwnd,
|
|
raw_x,
|
|
raw_y,
|
|
current_point=(uia_x, uia_y),
|
|
current_target=target or {},
|
|
)
|
|
dpi_meta.update(probe_meta)
|
|
target_scope = dict((target or {}).get("window_scope") or {})
|
|
atlas_class = str(target_scope.get("class_name") or "")
|
|
if self._is_root_or_oversized(target or {}) and atlas_class:
|
|
atlas_points = [(int(uia_x), int(uia_y)), (raw_x, raw_y)]
|
|
for candidate in probe_meta.get("uia_coordinate_candidates") or []:
|
|
if candidate.get("x") is not None and candidate.get("y") is not None:
|
|
atlas_points.append((int(candidate["x"]), int(candidate["y"])))
|
|
atlas_target = self._atlas_hit(atlas_class, atlas_points)
|
|
if atlas_target:
|
|
atlas_target["window_scope"] = dict(target_scope)
|
|
target = atlas_target
|
|
dpi_meta["coordinate_selection"] = "persisted_element_atlas"
|
|
dpi_meta["uia_coordinates"] = {"x": uia_x, "y": uia_y}
|
|
self._last_uia_point = (uia_x, uia_y)
|
|
self._last_raw_point = (raw_x, raw_y)
|
|
self._last_root_hwnd = int(root or hwnd or 0)
|
|
self._pending_target = target or {}
|
|
if self._pending_target and not self._is_root_or_oversized(self._pending_target):
|
|
self._last_actionable_target = dict(self._pending_target)
|
|
if self._is_text_entry(self._pending_target):
|
|
self._bind_text_target(self._pending_target)
|
|
self._seed_focus_baseline(self._pending_target)
|
|
meta: dict[str, Any] = {
|
|
"input_backend": self.health().get("backend"),
|
|
"injected": bool((source_meta or {}).get("injected")),
|
|
"source_time_ms": (source_meta or {}).get("source_time_ms"),
|
|
**dpi_meta,
|
|
}
|
|
if err:
|
|
meta["element_from_point_error"] = err
|
|
meta["raw_coordinates"] = {"x": x, "y": y}
|
|
# Coordinates saved as evidence only — never auto primary locator.
|
|
meta["coordinates_are_not_primary_locator"] = True
|
|
self.emit(
|
|
RawUserEvent(
|
|
kind="click",
|
|
surface="windows",
|
|
button=button,
|
|
x=uia_x,
|
|
y=uia_y,
|
|
target=self._pending_target,
|
|
meta=meta,
|
|
)
|
|
)
|
|
|
|
@staticmethod
|
|
def _uia_point_for_window(
|
|
hwnd: int,
|
|
x: int,
|
|
y: int,
|
|
*,
|
|
coordinate_space: str,
|
|
) -> tuple[int, int, dict[str, Any]]:
|
|
"""Map WH_*_LL physical coordinates into the DPI-virtualized UIA space."""
|
|
meta: dict[str, Any] = {
|
|
"raw_coordinates_physical": {"x": int(x), "y": int(y)},
|
|
"source_coordinate_space": coordinate_space,
|
|
}
|
|
if coordinate_space != "physical_screen" or not hwnd:
|
|
meta["uia_coordinates"] = {"x": int(x), "y": int(y)}
|
|
return int(x), int(y), meta
|
|
try:
|
|
# UIAutomation providers do not necessarily use the same coordinate
|
|
# space as a DPI-aware WH_MOUSE_LL hook. In particular WeCom can
|
|
# expose a 1440x900 UIA root while the hook and DWM both report a
|
|
# 2880x1800 physical window. Comparing GetWindowRect with DWM alone
|
|
# therefore incorrectly produces a scale of 1. Prefer the actual
|
|
# UIA root rectangle as the destination coordinate space.
|
|
physical = wintypes.RECT()
|
|
hr = ctypes.windll.dwmapi.DwmGetWindowAttribute(
|
|
int(hwnd), 9, ctypes.byref(physical), ctypes.sizeof(physical)
|
|
)
|
|
if hr == 0:
|
|
try:
|
|
import uiautomation as auto
|
|
|
|
root = auto.ControlFromHandle(int(hwnd))
|
|
rr = root.BoundingRectangle if root is not None else None
|
|
rw = int(rr.right - rr.left) if rr is not None else 0
|
|
rh = int(rr.bottom - rr.top) if rr is not None else 0
|
|
pw = int(physical.right - physical.left)
|
|
ph = int(physical.bottom - physical.top)
|
|
if min(rw, rh, pw, ph) > 0:
|
|
ux = round(int(rr.left) + (int(x) - physical.left) * rw / pw)
|
|
uy = round(int(rr.top) + (int(y) - physical.top) * rh / ph)
|
|
sx = pw / rw
|
|
sy = ph / rh
|
|
meta.update(
|
|
{
|
|
"uia_coordinates": {"x": ux, "y": uy},
|
|
"dpi_scale": {"x": round(sx, 4), "y": round(sy, 4)},
|
|
"coordinate_conversion": "dwm_physical_to_uia_root",
|
|
}
|
|
)
|
|
return ux, uy, meta
|
|
except Exception as exc:
|
|
meta["uia_root_coordinate_error"] = f"{type(exc).__name__}: {exc}"
|
|
|
|
logical = wintypes.RECT()
|
|
ctypes.windll.user32.GetWindowRect(int(hwnd), ctypes.byref(logical))
|
|
# Compatibility fallback for providers whose root rectangle cannot
|
|
# be queried. DWMWA_EXTENDED_FRAME_BOUNDS is in physical pixels.
|
|
lw = int(logical.right - logical.left)
|
|
lh = int(logical.bottom - logical.top)
|
|
pw = int(physical.right - physical.left)
|
|
ph = int(physical.bottom - physical.top)
|
|
if hr == 0 and min(lw, lh, pw, ph) > 0:
|
|
sx = pw / lw
|
|
sy = ph / lh
|
|
ux = round(logical.left + (int(x) - physical.left) / sx)
|
|
uy = round(logical.top + (int(y) - physical.top) / sy)
|
|
meta.update(
|
|
{
|
|
"uia_coordinates": {"x": ux, "y": uy},
|
|
"dpi_scale": {"x": round(sx, 4), "y": round(sy, 4)},
|
|
"coordinate_conversion": "dwm_physical_to_getwindowrect_logical",
|
|
}
|
|
)
|
|
return ux, uy, meta
|
|
except Exception as exc:
|
|
meta["coordinate_conversion_error"] = f"{type(exc).__name__}: {exc}"
|
|
meta["uia_coordinates"] = {"x": int(x), "y": int(y)}
|
|
meta["coordinate_conversion"] = "identity_fallback"
|
|
return int(x), int(y), meta
|
|
|
|
@staticmethod
|
|
def _target_quality(target: dict[str, Any]) -> tuple[int, int, int, int, int]:
|
|
bounds = dict(target.get("bounds") or {})
|
|
width = max(0, int(bounds.get("width") or 0))
|
|
height = max(0, int(bounds.get("height") or 0))
|
|
area = width * height
|
|
control_type = str(target.get("control_type") or "")
|
|
text = " ".join(
|
|
str(target.get(key) or "")
|
|
for key in ("name", "automation_id", "class_name", "control_type")
|
|
).lower()
|
|
oversized = control_type == "WindowControl" or area > 1_000_000
|
|
interactive = control_type in {
|
|
"ButtonControl", "EditControl", "TreeItemControl", "ListItemControl",
|
|
"MenuItemControl", "CheckBoxControl", "RadioButtonControl",
|
|
"ComboBoxControl", "TabItemControl", "HyperlinkControl",
|
|
}
|
|
semantic = any(
|
|
token in text
|
|
for token in (
|
|
"search", "edit", "input", "message", "send", "contact",
|
|
"buddy", "result", "comment", "reply",
|
|
)
|
|
)
|
|
return (
|
|
0 if oversized else 1,
|
|
1 if interactive else 0,
|
|
1 if semantic else 0,
|
|
len(target.get("parent_chain") or []),
|
|
-area,
|
|
)
|
|
|
|
def _probe_mixed_uia_coordinates(
|
|
self,
|
|
hwnd: int,
|
|
raw_x: int,
|
|
raw_y: int,
|
|
*,
|
|
current_point: tuple[int, int],
|
|
current_target: dict[str, Any],
|
|
) -> tuple[dict[str, Any], int, int, dict[str, Any]]:
|
|
"""Resolve providers whose root and descendants use different DPI spaces.
|
|
|
|
WeCom exposes a physical-size root but logical-size descendants. Only
|
|
probe alternative points when the direct hit is oversized and belongs
|
|
to a known mixed-space window, so standard desktop applications retain
|
|
their normal UIA hit-testing path.
|
|
"""
|
|
scope = dict(current_target.get("window_scope") or {})
|
|
window_class = str(scope.get("class_name") or current_target.get("class_name") or "")
|
|
if (
|
|
not self._is_root_or_oversized(current_target)
|
|
or window_class not in {"WeWorkWindow", "SearchResultWindow2"}
|
|
):
|
|
return current_target, current_point[0], current_point[1], {}
|
|
|
|
candidates: list[tuple[str, int, int, float]] = [
|
|
("direct", int(current_point[0]), int(current_point[1]), 1.0)
|
|
]
|
|
try:
|
|
get_dpi = getattr(ctypes.windll.user32, "GetDpiForWindow", None)
|
|
dpi = int(get_dpi(int(hwnd)) or 96) if get_dpi else 96
|
|
scale = max(1.0, float(dpi) / 96.0)
|
|
if scale > 1.05:
|
|
candidates.append(
|
|
(
|
|
"window_dpi_absolute",
|
|
round(int(raw_x) / scale),
|
|
round(int(raw_y) / scale),
|
|
scale,
|
|
)
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
unique: list[tuple[str, int, int, float]] = []
|
|
seen: set[tuple[int, int]] = set()
|
|
for item in candidates:
|
|
point = (item[1], item[2])
|
|
if point not in seen:
|
|
seen.add(point)
|
|
unique.append(item)
|
|
|
|
expected_pid = int(scope.get("pid") or 0)
|
|
expected_class = window_class
|
|
best_target = current_target
|
|
best_point = current_point
|
|
best_label = "direct"
|
|
best_scale = 1.0
|
|
best_quality = self._target_quality(current_target)
|
|
attempts: list[dict[str, Any]] = []
|
|
for label, px, py, scale in unique[1:]:
|
|
candidate, _err = self._element_from_point(px, py)
|
|
candidate_scope = dict((candidate or {}).get("window_scope") or {})
|
|
same_scope = bool(
|
|
candidate
|
|
and (not expected_pid or int(candidate_scope.get("pid") or 0) == expected_pid)
|
|
and (
|
|
not expected_class
|
|
or str(candidate_scope.get("class_name") or "") == expected_class
|
|
)
|
|
)
|
|
quality = self._target_quality(candidate or {}) if same_scope else (-1, 0, 0, 0, 0)
|
|
attempts.append(
|
|
{
|
|
"strategy": label,
|
|
"x": px,
|
|
"y": py,
|
|
"scale": round(scale, 4),
|
|
"target": str((candidate or {}).get("name") or ""),
|
|
"control_type": str((candidate or {}).get("control_type") or ""),
|
|
"same_scope": same_scope,
|
|
}
|
|
)
|
|
if same_scope and quality > best_quality:
|
|
best_target = candidate
|
|
best_point = (px, py)
|
|
best_label = label
|
|
best_scale = scale
|
|
best_quality = quality
|
|
|
|
return (
|
|
best_target,
|
|
int(best_point[0]),
|
|
int(best_point[1]),
|
|
{
|
|
"uia_coordinate_candidates": attempts,
|
|
"coordinate_selection": best_label,
|
|
"mixed_coordinate_scale": round(best_scale, 4),
|
|
},
|
|
)
|
|
|
|
def _on_key_down(self, packet: dict[str, Any]) -> None:
|
|
key = _SPECIAL_KEYS.get(int(packet.get("vk") or 0), "")
|
|
# Never call UIA here. Physical keys can arrive every few milliseconds;
|
|
# this consumer must remain faster than the hook producer even when the
|
|
# target accessibility provider is frozen for seconds.
|
|
target = self._cached_focus_snapshot()
|
|
if not self._is_text_entry(target):
|
|
fallback = dict(self._last_text_entry_target or {})
|
|
if fallback and (not target or self._last_point_matches_target(fallback)):
|
|
target = fallback
|
|
if self._ime_composing:
|
|
# Enter/Escape inside an IME candidate/composition window confirms
|
|
# or cancels composition; it is not a business-level key action.
|
|
# Keep the transaction attached to the last real editor.
|
|
if target:
|
|
self._typing_active = True
|
|
self._last_key_at = time.monotonic()
|
|
self._emit_typing_activity(target, packet)
|
|
self._uia_request.set()
|
|
return
|
|
if not key:
|
|
self._typing_active = True
|
|
self._last_key_at = time.monotonic()
|
|
self._emit_typing_activity(target, packet)
|
|
self._uia_request.set()
|
|
return
|
|
if key in {"Backspace", "Delete"} and self._is_text_entry(target):
|
|
# Editing keys are part of the resulting value transaction. The
|
|
# next value poll records one complete fill instead of key spam.
|
|
self._typing_active = True
|
|
self._last_key_at = time.monotonic()
|
|
self._emit_typing_activity(target, packet)
|
|
self._uia_request.set()
|
|
return
|
|
# Enter/Tab/click are transaction boundaries: publish the latest
|
|
# sampled value before the boundary action, preserving causal order.
|
|
self._emit_cached_value_change(force=True)
|
|
self.emit(
|
|
RawUserEvent(
|
|
kind="key",
|
|
surface="windows",
|
|
key=key,
|
|
target=target,
|
|
meta={
|
|
"input_backend": self.health().get("backend"),
|
|
"injected": bool(packet.get("injected")),
|
|
"source_time_ms": packet.get("source_time_ms"),
|
|
},
|
|
)
|
|
)
|
|
self._uia_request.set()
|
|
|
|
def _emit_typing_activity(
|
|
self, target: dict[str, Any], packet: dict[str, Any]
|
|
) -> None:
|
|
"""Journal one marker per text transaction without persisting key text.
|
|
|
|
The marker lets diagnostics distinguish "no typing happened" from
|
|
"typing happened but the provider never exposed a value change". It
|
|
is intentionally not reduced into an Action.
|
|
"""
|
|
target = dict(target or self._last_text_entry_target or {})
|
|
identity = "|".join(
|
|
str(target.get(key) or "")
|
|
for key in ("automation_id", "name", "control_type")
|
|
)
|
|
scope = dict(target.get("window_scope") or {})
|
|
identity = f"{scope.get('hwnd') or 0}|{identity}"
|
|
now = time.monotonic()
|
|
if (
|
|
identity == self._last_typing_activity_identity
|
|
and now - self._last_typing_activity_at < 0.8
|
|
):
|
|
self._last_typing_activity_at = now
|
|
return
|
|
self._last_typing_activity_identity = identity
|
|
self._last_typing_activity_at = now
|
|
self.emit(
|
|
RawUserEvent(
|
|
kind="typing_activity",
|
|
surface="windows",
|
|
target=target,
|
|
meta={
|
|
"input_backend": self.health().get("backend"),
|
|
"injected": bool(packet.get("injected")),
|
|
"source_time_ms": packet.get("source_time_ms"),
|
|
"content_recorded": False,
|
|
"unresolved_target": not bool(target),
|
|
"root_hwnd": int(self._last_root_hwnd or 0),
|
|
"last_uia_point": (
|
|
{"x": self._last_uia_point[0], "y": self._last_uia_point[1]}
|
|
if self._last_uia_point else {}
|
|
),
|
|
},
|
|
)
|
|
)
|
|
|
|
def _focused_control_snapshot(self) -> dict[str, Any]:
|
|
try:
|
|
import desktop_uia as uia
|
|
uia.ensure_com_sta()
|
|
import uiautomation as auto
|
|
ctrl = auto.GetFocusedControl()
|
|
info = self._control_snapshot(ctrl) if ctrl is not None else {}
|
|
self._ime_composing = self._is_transient_input_target(info)
|
|
unusable = self._is_root_or_oversized(info) or self._is_transient_input_target(info)
|
|
if unusable and self._last_uia_point:
|
|
try:
|
|
point_ctrl = auto.ControlFromPoint(*self._last_uia_point)
|
|
point_info = self._control_snapshot(
|
|
point_ctrl, x=self._last_uia_point[0], y=self._last_uia_point[1]
|
|
)
|
|
if not self._is_root_or_oversized(point_info):
|
|
info = point_info
|
|
except Exception:
|
|
pass
|
|
if (
|
|
(self._is_root_or_oversized(info) or self._is_transient_input_target(info))
|
|
and self._last_raw_point
|
|
and self._last_uia_point
|
|
and self._last_root_hwnd
|
|
):
|
|
try:
|
|
probed, _x, _y, _meta = self._probe_mixed_uia_coordinates(
|
|
self._last_root_hwnd,
|
|
self._last_raw_point[0],
|
|
self._last_raw_point[1],
|
|
current_point=self._last_uia_point,
|
|
current_target=info,
|
|
)
|
|
if not self._is_root_or_oversized(probed) and not self._is_transient_input_target(probed):
|
|
info = probed
|
|
except Exception:
|
|
pass
|
|
if (
|
|
(self._is_root_or_oversized(info) or self._is_transient_input_target(info))
|
|
and self._last_text_entry_target
|
|
):
|
|
refreshed = self._snapshot_live_text_control()
|
|
if refreshed:
|
|
info = refreshed
|
|
if (
|
|
(self._is_root_or_oversized(info) or self._is_transient_input_target(info))
|
|
and self._last_actionable_target
|
|
and self._last_point_matches_target(self._last_actionable_target)
|
|
):
|
|
current_scope = dict(info.get("window_scope") or {})
|
|
last_scope = dict(self._last_actionable_target.get("window_scope") or {})
|
|
current_hwnd = int(current_scope.get("hwnd") or self._last_root_hwnd or 0)
|
|
last_hwnd = int(last_scope.get("hwnd") or 0)
|
|
if not current_hwnd or not last_hwnd or current_hwnd == last_hwnd:
|
|
info = dict(self._last_actionable_target)
|
|
if (
|
|
(self._is_root_or_oversized(info) or self._is_transient_input_target(info))
|
|
and self._last_uia_point
|
|
):
|
|
scope = dict(info.get("window_scope") or {})
|
|
class_name = str(scope.get("class_name") or "")
|
|
points = [self._last_uia_point]
|
|
if self._last_raw_point:
|
|
rx, ry = self._last_raw_point
|
|
points.extend(
|
|
[(rx, ry)]
|
|
+ [(round(rx / scale), round(ry / scale)) for scale in (1.25, 1.5, 1.75, 2.0, 2.5, 3.0)]
|
|
)
|
|
atlas_target = self._atlas_hit(class_name, points)
|
|
if atlas_target and self._is_text_entry(atlas_target):
|
|
atlas_target["window_scope"] = dict(scope)
|
|
self._bind_text_target(atlas_target)
|
|
info = self._snapshot_live_text_control() or atlas_target
|
|
if (
|
|
(self._is_root_or_oversized(info) or self._is_transient_input_target(info))
|
|
and self._last_text_entry_target
|
|
):
|
|
current_scope = dict(info.get("window_scope") or {})
|
|
editor_scope = dict(self._last_text_entry_target.get("window_scope") or {})
|
|
current_pid = int(current_scope.get("pid") or 0)
|
|
editor_pid = int(editor_scope.get("pid") or 0)
|
|
# IME windows have a different HWND but normally share the
|
|
# application's PID. Never let the IME HWND replace the
|
|
# business editor just because the provider probe is cold.
|
|
if not current_pid or not editor_pid or current_pid == editor_pid:
|
|
refreshed = self._snapshot_live_text_control()
|
|
info = refreshed or dict(self._last_text_entry_target)
|
|
if info and not self._is_root_or_oversized(info) and not self._is_transient_input_target(info):
|
|
self._last_actionable_target = dict(info)
|
|
if self._is_text_entry(info):
|
|
self._bind_text_target(info, resolve_live=False)
|
|
return info
|
|
# A transient OS surface must never cross the recorder boundary as
|
|
# an application target.
|
|
return {}
|
|
except Exception as exc:
|
|
self._record_error(exc)
|
|
return {}
|
|
|
|
def _on_mouse_up(self) -> None:
|
|
# Click already emitted on down (capture before UI disappears).
|
|
self._pending_target = {}
|
|
|
|
def _poll_focus_value(self) -> None:
|
|
try:
|
|
info = self._focused_control_snapshot()
|
|
if not info:
|
|
return
|
|
# Skip if focused control belongs to Inspector
|
|
hwnd = int(info.get("window_scope", {}).get("hwnd") or 0)
|
|
try:
|
|
exclude = set(self.exclude_hwnds() or set())
|
|
except Exception:
|
|
exclude = set()
|
|
if hwnd and hwnd in exclude:
|
|
return
|
|
fp = "|".join(
|
|
[
|
|
str(info.get("automation_id") or ""),
|
|
str(info.get("name") or ""),
|
|
str(info.get("control_type") or ""),
|
|
]
|
|
)
|
|
val = str(info.get("value") or "")
|
|
if fp == self._last_focus_fp and val == self._last_focus_value:
|
|
return
|
|
if self._last_focus_fp and fp == self._last_focus_fp and val != self._last_focus_value:
|
|
self.emit(
|
|
RawUserEvent(
|
|
kind="value_change",
|
|
surface="windows",
|
|
text=val,
|
|
target=info,
|
|
meta={"input_type": "password" if "password" in (info.get("name") or "").lower() else ""},
|
|
)
|
|
)
|
|
self._last_focus_fp = fp
|
|
self._last_focus_value = val
|
|
info["fingerprint"] = fp
|
|
except Exception:
|
|
return
|
|
|
|
@staticmethod
|
|
def _is_text_entry(info: dict[str, Any]) -> bool:
|
|
text = " ".join(
|
|
str(info.get(key) or "")
|
|
for key in ("name", "automation_id", "class_name", "control_type")
|
|
).lower()
|
|
return str(info.get("control_type") or "") == "EditControl" or any(
|
|
token in text
|
|
for token in ("edit", "input", "search", "compose", "message", "rich")
|
|
)
|
|
|
|
@staticmethod
|
|
def _is_transient_input_target(info: dict[str, Any]) -> bool:
|
|
text = " ".join(
|
|
str(value or "")
|
|
for value in (
|
|
info.get("name"),
|
|
info.get("class_name"),
|
|
(info.get("window_scope") or {}).get("class_name"),
|
|
(info.get("window_scope") or {}).get("title"),
|
|
)
|
|
).lower()
|
|
return any(
|
|
token in text
|
|
for token in (
|
|
"sopy_comp", "msctfime", "candidatewindow", "ime candidate",
|
|
"inputmethod", "input method", "compositionwindow",
|
|
"拼音", "候选词", "输入法",
|
|
)
|
|
)
|
|
|
|
def _last_point_matches_target(self, target: dict[str, Any]) -> bool:
|
|
"""Guard stale focus fallback with the last click's geometry."""
|
|
bounds = dict(target.get("bounds") or {})
|
|
try:
|
|
left = int(bounds.get("left"))
|
|
top = int(bounds.get("top"))
|
|
right = int(bounds.get("right"))
|
|
bottom = int(bounds.get("bottom"))
|
|
except (TypeError, ValueError):
|
|
return False
|
|
points: list[tuple[int, int]] = []
|
|
if self._last_uia_point:
|
|
points.append(self._last_uia_point)
|
|
if self._last_raw_point:
|
|
rx, ry = self._last_raw_point
|
|
points.append((rx, ry))
|
|
# Mixed-DPI providers commonly expose descendants at one of these
|
|
# Windows scale factors even when their root uses physical pixels.
|
|
for scale in (1.25, 1.5, 1.75, 2.0, 2.5, 3.0):
|
|
points.append((round(rx / scale), round(ry / scale)))
|
|
return any(left <= x <= right and top <= y <= bottom for x, y in points)
|
|
|
|
def _seed_focus_baseline(self, info: dict[str, Any]) -> None:
|
|
"""Remember the pre-input value at click time.
|
|
|
|
Without this seed, fast typing can finish before the first 400 ms focus
|
|
poll and the completed text is mistaken for the initial baseline.
|
|
"""
|
|
if not info or not self._is_text_entry(info):
|
|
return
|
|
self._bind_text_target(info)
|
|
self._last_focus_fp = "|".join(
|
|
[
|
|
str(info.get("automation_id") or ""),
|
|
str(info.get("name") or ""),
|
|
str(info.get("control_type") or ""),
|
|
]
|
|
)
|
|
self._last_focus_value = str(info.get("value") or "")
|
|
|
|
def learn_elements(
|
|
self,
|
|
rows: list[dict[str, Any]],
|
|
*,
|
|
window: Optional[dict[str, Any]] = None,
|
|
) -> None:
|
|
"""Publish a bounded, value-free element atlas to the event-time path."""
|
|
win = dict(window or {})
|
|
class_name = str(win.get("class_name") or win.get("chosen_class") or "")
|
|
if not class_name:
|
|
return
|
|
cleaned: list[dict[str, Any]] = []
|
|
for row in rows or []:
|
|
item = dict(row or {})
|
|
bounds = dict(item.get("bounds") or item.get("rect") or {})
|
|
if not bounds:
|
|
continue
|
|
item["bounds"] = bounds
|
|
item.pop("rect", None)
|
|
# Historical atlases are locators, not a store of user content.
|
|
item["value"] = ""
|
|
item["window_scope"] = {
|
|
"hwnd": int(win.get("hwnd") or 0),
|
|
"pid": int(win.get("pid") or 0),
|
|
"title": str(win.get("title") or ""),
|
|
"class_name": class_name,
|
|
}
|
|
cleaned.append(item)
|
|
if cleaned:
|
|
with self._atlas_lock:
|
|
self._element_atlas[class_name] = cleaned[-2500:]
|
|
|
|
def _atlas_hit(
|
|
self, class_name: str, points: list[tuple[int, int]]
|
|
) -> dict[str, Any]:
|
|
with self._atlas_lock:
|
|
rows = list(self._element_atlas.get(class_name) or [])
|
|
candidates: list[tuple[tuple[int, int, int, int], dict[str, Any]]] = []
|
|
for row in rows:
|
|
bounds = dict(row.get("bounds") or {})
|
|
try:
|
|
hit = any(
|
|
int(bounds.get("left")) <= x <= int(bounds.get("right"))
|
|
and int(bounds.get("top")) <= y <= int(bounds.get("bottom"))
|
|
for x, y in points
|
|
)
|
|
except (TypeError, ValueError):
|
|
continue
|
|
if not hit or row.get("is_offscreen") is True:
|
|
continue
|
|
name = str(row.get("name") or row.get("automation_id") or "").lower()
|
|
control_type = str(row.get("control_type") or "")
|
|
width = max(0, int(bounds.get("width") or 0))
|
|
height = max(0, int(bounds.get("height") or 0))
|
|
area = width * height
|
|
if not area or width < 4 or height < 4:
|
|
continue
|
|
exact_editor = name in {
|
|
"searchedit", "send_rich_edit", "search_box", "compose_box"
|
|
}
|
|
editor = self._is_text_entry(row) and not name.endswith("_layout")
|
|
semantic = any(
|
|
token in name
|
|
for token in ("search_name", "contact_name", "send", "edit", "input")
|
|
)
|
|
candidates.append(
|
|
((1 if exact_editor else 0, 1 if editor else 0, 1 if semantic else 0, -area), row)
|
|
)
|
|
return dict(max(candidates, default=((0, 0, 0, 0), {}), key=lambda x: x[0])[1])
|
|
|
|
def _bind_text_target(
|
|
self, info: dict[str, Any], *, resolve_live: bool = True
|
|
) -> None:
|
|
if not info or not self._is_text_entry(info):
|
|
return
|
|
self._last_text_entry_target = dict(info)
|
|
if not resolve_live or not self._last_root_hwnd:
|
|
return
|
|
name = str(info.get("name") or "").strip()
|
|
if not name:
|
|
return
|
|
try:
|
|
import uiautomation as auto
|
|
|
|
root = auto.ControlFromHandle(int(self._last_root_hwnd))
|
|
candidate = root.Control(searchDepth=24, Name=name)
|
|
if candidate.Exists(0.35, 0.02):
|
|
self._live_text_control = candidate
|
|
except Exception:
|
|
self._live_text_control = None
|
|
|
|
def _snapshot_live_text_control(self) -> dict[str, Any]:
|
|
if self._live_text_control is None:
|
|
return {}
|
|
try:
|
|
info = self._control_snapshot(self._live_text_control)
|
|
if info and self._is_text_entry(info):
|
|
return info
|
|
except Exception:
|
|
pass
|
|
self._live_text_control = None
|
|
return {}
|
|
|
|
@staticmethod
|
|
def _is_root_or_oversized(info: dict[str, Any]) -> bool:
|
|
bounds = dict(info.get("bounds") or {})
|
|
area = int(bounds.get("width") or 0) * int(bounds.get("height") or 0)
|
|
return str(info.get("control_type") or "") == "WindowControl" or area > 1_000_000
|
|
|
|
def _element_from_point(self, x: int, y: int) -> tuple[dict[str, Any], str]:
|
|
"""Minimal directed read — NO ensure_ui_ready / full warm."""
|
|
try:
|
|
import desktop_uia as uia
|
|
|
|
uia.ensure_com_sta()
|
|
import uiautomation as auto
|
|
|
|
ctrl = auto.ControlFromPoint(int(x), int(y))
|
|
if ctrl is None:
|
|
return {}, "ControlFromPoint returned None"
|
|
return self._control_snapshot(ctrl, x=x, y=y), ""
|
|
except Exception as exc:
|
|
return {}, f"{type(exc).__name__}: {exc}"
|
|
|
|
def _element_from_handle(self, hwnd: int) -> tuple[dict[str, Any], str]:
|
|
try:
|
|
import desktop_uia as uia
|
|
|
|
uia.ensure_com_sta()
|
|
import uiautomation as auto
|
|
|
|
ctrl = auto.ControlFromHandle(int(hwnd))
|
|
if ctrl is None:
|
|
return {}, "ControlFromHandle returned None"
|
|
return self._control_snapshot(ctrl), ""
|
|
except Exception as exc:
|
|
return {}, f"{type(exc).__name__}: {exc}"
|
|
|
|
@staticmethod
|
|
def _is_decorative_window_class(class_name: str) -> bool:
|
|
lowered = str(class_name or "").strip().lower()
|
|
return lowered in {
|
|
"shell_traywnd", "shell_secondarytraywnd", "progman", "workerw",
|
|
"ime", "msctfime ui", "tooltips_class32", "sopy_comp",
|
|
"perryshadowwnd",
|
|
} or any(
|
|
token in lowered
|
|
for token in ("shadowwnd", "dropshadow", "drop_shadow", "windowshadow")
|
|
)
|
|
|
|
@staticmethod
|
|
def _top_level_window_at_point(
|
|
x: int,
|
|
y: int,
|
|
*,
|
|
exclude_hwnds: set[int],
|
|
preferred_pid: int = 0,
|
|
) -> int:
|
|
"""Return the smallest visible top-level overlay containing a point.
|
|
|
|
This is a generic Win32 fallback for applications whose accessibility
|
|
provider reports the owner window from ``WindowFromPoint`` while an
|
|
owned popup/list is visibly on top.
|
|
"""
|
|
if os.name != "nt":
|
|
return 0
|
|
user32 = ctypes.windll.user32
|
|
candidates: list[tuple[int, int]] = []
|
|
@ctypes.WINFUNCTYPE(ctypes.c_bool, wintypes.HWND, wintypes.LPARAM)
|
|
def callback(hwnd, _lparam):
|
|
handle = int(hwnd or 0)
|
|
if not handle or handle in exclude_hwnds or not user32.IsWindowVisible(handle):
|
|
return True
|
|
pid_ref = ctypes.c_ulong()
|
|
user32.GetWindowThreadProcessId(handle, ctypes.byref(pid_ref))
|
|
pid = int(pid_ref.value or 0)
|
|
if pid == os.getpid() or (preferred_pid and pid != preferred_pid):
|
|
return True
|
|
buf = ctypes.create_unicode_buffer(256)
|
|
user32.GetClassNameW(handle, buf, len(buf))
|
|
class_name = buf.value
|
|
if WindowsRecorder._is_decorative_window_class(class_name):
|
|
return True
|
|
rect = wintypes.RECT()
|
|
if not user32.GetWindowRect(handle, ctypes.byref(rect)):
|
|
return True
|
|
if not (rect.left <= int(x) <= rect.right and rect.top <= int(y) <= rect.bottom):
|
|
return True
|
|
width = max(0, int(rect.right - rect.left))
|
|
height = max(0, int(rect.bottom - rect.top))
|
|
if width < 8 or height < 8:
|
|
return True
|
|
candidates.append((width * height, handle))
|
|
return True
|
|
|
|
try:
|
|
user32.EnumWindows(callback, 0)
|
|
except Exception:
|
|
return 0
|
|
return min(candidates, default=(0, 0))[1]
|
|
|
|
def _control_snapshot(
|
|
self, ctrl: Any, *, x: Optional[int] = None, y: Optional[int] = None
|
|
) -> dict[str, Any]:
|
|
def safe(fn, default=""):
|
|
try:
|
|
return fn()
|
|
except Exception:
|
|
return default
|
|
|
|
name = str(safe(lambda: ctrl.Name, "") or "")
|
|
value = ""
|
|
try:
|
|
value = str(ctrl.GetValuePattern().Value)
|
|
except Exception:
|
|
value = str(safe(lambda: getattr(ctrl, "Value", ""), "") or "")
|
|
if not value:
|
|
try:
|
|
value = str(ctrl.GetTextPattern().DocumentRange.GetText(-1) or "")
|
|
except Exception:
|
|
pass
|
|
if not value:
|
|
try:
|
|
value = str(ctrl.GetLegacyIAccessiblePattern().Value or "")
|
|
except Exception:
|
|
pass
|
|
aid = str(safe(lambda: ctrl.AutomationId, "") or "")
|
|
ctype = str(safe(lambda: ctrl.ControlTypeName, "") or "")
|
|
class_name = str(safe(lambda: ctrl.ClassName, "") or "")
|
|
framework = str(safe(lambda: ctrl.FrameworkId, "") or "")
|
|
enabled = safe(lambda: bool(ctrl.IsEnabled), None)
|
|
offscreen = safe(lambda: bool(ctrl.IsOffscreen), None)
|
|
rect: dict[str, int] = {}
|
|
try:
|
|
r = ctrl.BoundingRectangle
|
|
rect = {
|
|
"left": int(r.left),
|
|
"top": int(r.top),
|
|
"right": int(r.right),
|
|
"bottom": int(r.bottom),
|
|
"width": int(r.right - r.left),
|
|
"height": int(r.bottom - r.top),
|
|
}
|
|
except Exception:
|
|
rect = {}
|
|
|
|
parent_chain: list[dict[str, str]] = []
|
|
try:
|
|
p = ctrl.GetParentControl()
|
|
depth = 0
|
|
while p is not None and depth < 8:
|
|
parent_chain.append(
|
|
{
|
|
"name": str(getattr(p, "Name", "") or ""),
|
|
"automation_id": str(getattr(p, "AutomationId", "") or ""),
|
|
"control_type": str(getattr(p, "ControlTypeName", "") or ""),
|
|
"class_name": str(getattr(p, "ClassName", "") or ""),
|
|
}
|
|
)
|
|
p = p.GetParentControl()
|
|
depth += 1
|
|
except Exception:
|
|
pass
|
|
|
|
hwnd = 0
|
|
pid = 0
|
|
wtitle = ""
|
|
wclass = ""
|
|
try:
|
|
hwnd = int(ctrl.NativeWindowHandle or 0)
|
|
if not hwnd:
|
|
# walk up for hwnd
|
|
for item in parent_chain:
|
|
del item
|
|
p = ctrl
|
|
for _ in range(12):
|
|
h = int(getattr(p, "NativeWindowHandle", 0) or 0)
|
|
if h:
|
|
hwnd = h
|
|
break
|
|
p = p.GetParentControl()
|
|
if p is None:
|
|
break
|
|
if hwnd:
|
|
import desktop_uia as uia
|
|
|
|
info = uia.window_info_from_hwnd(hwnd) if hasattr(uia, "window_info_from_hwnd") else None
|
|
if info is None:
|
|
# fallback via GetWindowText
|
|
import ctypes
|
|
|
|
user32 = ctypes.windll.user32
|
|
buf = ctypes.create_unicode_buffer(512)
|
|
user32.GetWindowTextW(hwnd, buf, 512)
|
|
wtitle = buf.value
|
|
user32.GetClassNameW(hwnd, buf, 512)
|
|
wclass = buf.value
|
|
pid_c = ctypes.c_ulong()
|
|
user32.GetWindowThreadProcessId(hwnd, ctypes.byref(pid_c))
|
|
pid = int(pid_c.value)
|
|
else:
|
|
wtitle = info.title
|
|
wclass = info.class_name
|
|
pid = info.pid
|
|
hwnd = info.hwnd
|
|
except Exception:
|
|
pass
|
|
|
|
snap = {
|
|
"name": name,
|
|
"value": value,
|
|
"automation_id": aid,
|
|
"control_type": ctype,
|
|
"class_name": class_name,
|
|
"framework_id": framework,
|
|
"is_enabled": enabled,
|
|
"is_offscreen": offscreen,
|
|
"bounds": rect,
|
|
"parent_chain": parent_chain,
|
|
"window_scope": {
|
|
"hwnd": hwnd,
|
|
"pid": pid,
|
|
"title": wtitle,
|
|
"class_name": wclass,
|
|
},
|
|
}
|
|
if x is not None:
|
|
snap["capture_point"] = {"x": x, "y": y}
|
|
return snap
|
|
|
|
@staticmethod
|
|
def _cursor_pos(user32) -> tuple[int, int]:
|
|
import ctypes
|
|
from ctypes import wintypes
|
|
|
|
class POINT(ctypes.Structure):
|
|
_fields_ = [("x", wintypes.LONG), ("y", wintypes.LONG)]
|
|
|
|
pt = POINT()
|
|
user32.GetCursorPos(ctypes.byref(pt))
|
|
return int(pt.x), int(pt.y)
|
|
|
|
@staticmethod
|
|
def _make_point(x: int, y: int):
|
|
import ctypes
|
|
from ctypes import wintypes
|
|
|
|
class POINT(ctypes.Structure):
|
|
_fields_ = [("x", wintypes.LONG), ("y", wintypes.LONG)]
|
|
|
|
return POINT(int(x), int(y))
|