chendelian
633ecff4e2
Keep the last in-template copy on main so history can restore it; product continues in jiangchang-ui-recorder. Co-authored-by: Cursor <cursoragent@cursor.com>
1732 lines
58 KiB
Python
1732 lines
58 KiB
Python
"""UI Automation helpers: attach, control-from-point, tree dump."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import ctypes
|
||
import time
|
||
from ctypes import wintypes
|
||
from dataclasses import asdict, dataclass, field
|
||
from typing import Any, Optional
|
||
|
||
user32 = ctypes.windll.user32
|
||
|
||
WM_GETOBJECT = 0x003D
|
||
OBJID_WINDOW = 0x00000000
|
||
OBJID_CLIENT = 0xFFFFFFFC
|
||
OBJID_SYSMENU = 0xFFFFFFFF
|
||
# UIA root object id ((DWORD)-25) — forces UI Automation provider, not just MSAA.
|
||
UIA_ROOT_OBJECT_ID = ctypes.c_long(0xFFFFFFE7).value
|
||
SW_RESTORE = 9
|
||
SW_SHOW = 5
|
||
SW_MAXIMIZE = 3
|
||
GA_ROOT = 2
|
||
GW_HWNDNEXT = 2
|
||
GW_OWNER = 4
|
||
GWL_EXSTYLE = -20
|
||
WS_EX_TOOLWINDOW = 0x00000080
|
||
WS_EX_NOACTIVATE = 0x08000000
|
||
DWMWA_CLOAKED = 14
|
||
|
||
|
||
@dataclass
|
||
class WindowInfo:
|
||
hwnd: int
|
||
pid: int
|
||
title: str
|
||
class_name: str
|
||
process_name: str = ""
|
||
|
||
def to_dict(self) -> dict[str, Any]:
|
||
return asdict(self)
|
||
|
||
|
||
@dataclass
|
||
class ControlInfo:
|
||
name: str = ""
|
||
value: str = ""
|
||
automation_id: str = ""
|
||
control_type: str = ""
|
||
class_name: str = ""
|
||
framework_id: str = ""
|
||
localized_control_type: str = ""
|
||
is_enabled: Optional[bool] = None
|
||
is_offscreen: Optional[bool] = None
|
||
rect: dict[str, int] = field(default_factory=dict)
|
||
depth: int = 0
|
||
parent_chain: list[dict[str, str]] = field(default_factory=list)
|
||
path_hint: str = ""
|
||
|
||
def to_dict(self) -> dict[str, Any]:
|
||
return asdict(self)
|
||
|
||
|
||
def ensure_uia() -> None:
|
||
try:
|
||
import uiautomation # noqa: F401
|
||
except ImportError as exc:
|
||
raise RuntimeError(
|
||
"缺少 uiautomation。请先: pip install uiautomation>=2.0.18"
|
||
) from exc
|
||
|
||
|
||
def ensure_com_sta() -> None:
|
||
"""
|
||
UIAutomation is STA-friendly. Worker threads must enter an apartment
|
||
or FindAll often returns an empty shell (findall=2~3).
|
||
"""
|
||
try:
|
||
import comtypes
|
||
|
||
try:
|
||
comtypes.CoInitializeEx(comtypes.COINIT_APARTMENTTHREADED)
|
||
except OSError:
|
||
# Already initialized on this thread (or MTA) — ignore.
|
||
pass
|
||
except Exception:
|
||
try:
|
||
comtypes.CoInitialize()
|
||
except Exception:
|
||
pass
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def window_area(hwnd: int) -> int:
|
||
rect = _window_rect(int(hwnd))
|
||
if not rect:
|
||
return 0
|
||
return max(0, int(rect.get("width") or 0)) * max(0, int(rect.get("height") or 0))
|
||
|
||
|
||
def _force_foreground(hwnd: int) -> bool:
|
||
"""AttachThreadInput + SetForeground — more reliable than bare SetForeground."""
|
||
hwnd = int(hwnd)
|
||
try:
|
||
fg = int(user32.GetForegroundWindow() or 0)
|
||
tid_fg = int(user32.GetWindowThreadProcessId(fg, None) or 0) if fg else 0
|
||
tid_t = int(user32.GetWindowThreadProcessId(hwnd, None) or 0)
|
||
attached = False
|
||
if tid_fg and tid_t and tid_fg != tid_t:
|
||
attached = bool(user32.AttachThreadInput(tid_fg, tid_t, True))
|
||
try:
|
||
user32.BringWindowToTop(hwnd)
|
||
ok = bool(user32.SetForegroundWindow(hwnd))
|
||
try:
|
||
user32.SetFocus(hwnd)
|
||
except Exception:
|
||
pass
|
||
return ok
|
||
finally:
|
||
if attached:
|
||
user32.AttachThreadInput(tid_fg, tid_t, False)
|
||
except Exception:
|
||
try:
|
||
return bool(user32.SetForegroundWindow(hwnd))
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def activate_main_hwnd(hwnd: int, *, maximize: bool = False) -> dict[str, Any]:
|
||
"""
|
||
Light activation for capture:
|
||
- restore only if minimized
|
||
- maximize only when explicitly requested (Inspector default: off)
|
||
- SetForeground only if not already foreground (avoid flash)
|
||
Only call on the chosen main/capture hwnd — never on every child.
|
||
"""
|
||
hwnd = int(hwnd)
|
||
info: dict[str, Any] = {
|
||
"hwnd": hwnd,
|
||
"iconic": False,
|
||
"zoomed_before": False,
|
||
"maximized": False,
|
||
"foreground": False,
|
||
"already_foreground": False,
|
||
"maximize_requested": bool(maximize),
|
||
}
|
||
try:
|
||
info["iconic"] = bool(user32.IsIconic(hwnd))
|
||
info["zoomed_before"] = bool(user32.IsZoomed(hwnd))
|
||
except Exception:
|
||
pass
|
||
|
||
if info["iconic"]:
|
||
user32.ShowWindow(hwnd, SW_RESTORE)
|
||
time.sleep(0.15)
|
||
if maximize and not info["zoomed_before"]:
|
||
user32.ShowWindow(hwnd, SW_MAXIMIZE)
|
||
time.sleep(0.2)
|
||
|
||
try:
|
||
already = int(user32.GetForegroundWindow() or 0) == hwnd
|
||
except Exception:
|
||
already = False
|
||
info["already_foreground"] = already
|
||
if already:
|
||
info["foreground"] = True
|
||
else:
|
||
info["foreground"] = _force_foreground(hwnd)
|
||
time.sleep(0.25 if maximize else 0.15)
|
||
try:
|
||
info["maximized"] = bool(user32.IsZoomed(hwnd))
|
||
except Exception:
|
||
pass
|
||
return info
|
||
|
||
|
||
def is_provider_cold(hwnd: int, *, findall: Optional[int] = None) -> bool:
|
||
"""True when WM_GETOBJECT CLIENT is 0 and FindAll is an empty shell."""
|
||
hwnd = int(hwnd)
|
||
n = int(findall) if findall is not None else findall_count_for_hwnd(hwnd)
|
||
if n < 0:
|
||
n = 0
|
||
if n >= 8:
|
||
return False
|
||
try:
|
||
res = int(user32.SendMessageW(hwnd, WM_GETOBJECT, 0, OBJID_CLIENT) or 0)
|
||
except Exception:
|
||
res = 0
|
||
return res == 0 and n < 8
|
||
|
||
|
||
def is_mainish_window(info: WindowInfo) -> bool:
|
||
"""Large primary app window (safe to maximize)."""
|
||
cls = info.class_name or ""
|
||
if cls in {"SearchResultWindow2", "IME", "MSCTFIME UI", "tooltips_class32"}:
|
||
return False
|
||
if "SearchResult" in cls:
|
||
return False
|
||
area = window_area(info.hwnd)
|
||
if area >= 500 * 400:
|
||
return True
|
||
if "WeWork" in cls or "Chrome_WidgetWin" in cls:
|
||
return area >= 300 * 200
|
||
return bool((info.title or "").strip()) and area >= 600 * 400
|
||
|
||
|
||
def largest_window(candidates: list[WindowInfo]) -> Optional[WindowInfo]:
|
||
if not candidates:
|
||
return None
|
||
return max(candidates, key=lambda c: window_area(c.hwnd))
|
||
|
||
|
||
def _safe_str(value: Any) -> str:
|
||
if value is None:
|
||
return ""
|
||
try:
|
||
return str(value).strip()
|
||
except Exception:
|
||
return ""
|
||
|
||
|
||
def _rect_dict(ctrl) -> dict[str, int]:
|
||
try:
|
||
r = ctrl.BoundingRectangle
|
||
return {
|
||
"left": int(r.left),
|
||
"top": int(r.top),
|
||
"right": int(r.right),
|
||
"bottom": int(r.bottom),
|
||
"width": int(r.width()),
|
||
"height": int(r.height()),
|
||
}
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
def _read_value(ctrl) -> str:
|
||
# Align with production skills: Value often lives in ValueValueProperty.
|
||
try:
|
||
import uiautomation as auto
|
||
|
||
raw = ctrl.GetPropertyValue(auto.PropertyId.ValueValueProperty)
|
||
s = _safe_str(raw)
|
||
if s:
|
||
return s
|
||
except Exception:
|
||
pass
|
||
try:
|
||
pattern = ctrl.GetValuePattern()
|
||
if pattern is not None:
|
||
s = _safe_str(pattern.Value)
|
||
if s:
|
||
return s
|
||
except Exception:
|
||
pass
|
||
for attr in ("Value", "LegacyIAccessibleValue", "HelpText"):
|
||
try:
|
||
s = _safe_str(getattr(ctrl, attr, None))
|
||
if s:
|
||
return s
|
||
except Exception:
|
||
continue
|
||
return ""
|
||
|
||
|
||
def _control_type_name(ctrl) -> str:
|
||
try:
|
||
ct = ctrl.ControlTypeName
|
||
if ct:
|
||
return _safe_str(ct)
|
||
except Exception:
|
||
pass
|
||
try:
|
||
return _safe_str(ctrl.ControlType)
|
||
except Exception:
|
||
return ""
|
||
|
||
|
||
def describe_control(ctrl, *, depth: int = 0, parent_chain: Optional[list] = None) -> ControlInfo:
|
||
info = ControlInfo(
|
||
name=_safe_str(getattr(ctrl, "Name", None)),
|
||
value=_read_value(ctrl),
|
||
automation_id=_safe_str(getattr(ctrl, "AutomationId", None)),
|
||
control_type=_control_type_name(ctrl),
|
||
class_name=_safe_str(getattr(ctrl, "ClassName", None)),
|
||
framework_id=_safe_str(getattr(ctrl, "FrameworkId", None)),
|
||
localized_control_type=_safe_str(getattr(ctrl, "LocalizedControlType", None)),
|
||
depth=depth,
|
||
parent_chain=list(parent_chain or []),
|
||
rect=_rect_dict(ctrl),
|
||
)
|
||
try:
|
||
info.is_enabled = bool(ctrl.IsEnabled)
|
||
except Exception:
|
||
info.is_enabled = None
|
||
try:
|
||
info.is_offscreen = bool(ctrl.IsOffscreen)
|
||
except Exception:
|
||
info.is_offscreen = None
|
||
|
||
bits = []
|
||
if info.control_type:
|
||
bits.append(info.control_type)
|
||
if info.name:
|
||
bits.append(f"Name={info.name!r}")
|
||
if info.value:
|
||
bits.append(f"Value={info.value!r}")
|
||
if info.automation_id:
|
||
bits.append(f"AutomationId={info.automation_id!r}")
|
||
info.path_hint = " | ".join(bits)
|
||
return info
|
||
|
||
|
||
def get_toplevel_hwnd(hwnd: int) -> int:
|
||
hwnd = int(hwnd)
|
||
if not hwnd:
|
||
return 0
|
||
root = int(user32.GetAncestor(hwnd, GA_ROOT) or 0)
|
||
return root or hwnd
|
||
|
||
|
||
def window_info_from_hwnd(hwnd: int) -> WindowInfo:
|
||
ensure_uia()
|
||
import uiautomation as auto
|
||
|
||
hwnd = get_toplevel_hwnd(int(hwnd))
|
||
if not hwnd or not user32.IsWindow(hwnd):
|
||
raise RuntimeError(f"无效窗口 HWND={hwnd}")
|
||
|
||
title_buf = ctypes.create_unicode_buffer(512)
|
||
user32.GetWindowTextW(hwnd, title_buf, 512)
|
||
class_buf = ctypes.create_unicode_buffer(256)
|
||
user32.GetClassNameW(hwnd, class_buf, 256)
|
||
|
||
pid = ctypes.c_ulong(0)
|
||
user32.GetWindowThreadProcessId(hwnd, ctypes.byref(pid))
|
||
|
||
process_name = ""
|
||
try:
|
||
root = auto.ControlFromHandle(hwnd)
|
||
process_name = _safe_str(getattr(root, "ProcessName", None))
|
||
except Exception:
|
||
pass
|
||
|
||
return WindowInfo(
|
||
hwnd=hwnd,
|
||
pid=int(pid.value),
|
||
title=_safe_str(title_buf.value),
|
||
class_name=_safe_str(class_buf.value),
|
||
process_name=process_name,
|
||
)
|
||
|
||
|
||
def get_foreground_window_info() -> WindowInfo:
|
||
hwnd = int(user32.GetForegroundWindow())
|
||
if not hwnd:
|
||
raise RuntimeError("当前没有前台窗口")
|
||
return window_info_from_hwnd(hwnd)
|
||
|
||
|
||
def is_inspector_window(info: WindowInfo, *, self_hwnd: int = 0) -> bool:
|
||
title = info.title or ""
|
||
if self_hwnd and int(info.hwnd) == int(self_hwnd):
|
||
return True
|
||
if self_hwnd and get_toplevel_hwnd(self_hwnd) == int(info.hwnd):
|
||
return True
|
||
if "UI Inspector" in title or "OpenClaw UI Inspector" in title:
|
||
return True
|
||
return False
|
||
|
||
|
||
def is_system_chrome_window(info: WindowInfo) -> bool:
|
||
cls = info.class_name or ""
|
||
return cls in {
|
||
"Shell_TrayWnd",
|
||
"Shell_SecondaryTrayWnd",
|
||
"NotifyIconOverflowWindow",
|
||
"Progman",
|
||
"WorkerW",
|
||
}
|
||
|
||
|
||
def _window_rect(hwnd: int) -> dict[str, int]:
|
||
rect = wintypes.RECT()
|
||
if not user32.GetWindowRect(int(hwnd), ctypes.byref(rect)):
|
||
return {}
|
||
return {
|
||
"left": int(rect.left),
|
||
"top": int(rect.top),
|
||
"right": int(rect.right),
|
||
"bottom": int(rect.bottom),
|
||
"width": int(rect.right - rect.left),
|
||
"height": int(rect.bottom - rect.top),
|
||
}
|
||
|
||
|
||
def _is_cloaked(hwnd: int) -> bool:
|
||
try:
|
||
dwmapi = ctypes.windll.dwmapi
|
||
cloaked = ctypes.c_int(0)
|
||
hr = dwmapi.DwmGetWindowAttribute(
|
||
wintypes.HWND(int(hwnd)),
|
||
ctypes.c_int(DWMWA_CLOAKED),
|
||
ctypes.byref(cloaked),
|
||
ctypes.sizeof(cloaked),
|
||
)
|
||
return hr == 0 and int(cloaked.value) != 0
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def _exstyle(hwnd: int) -> int:
|
||
return int(user32.GetWindowLongW(int(hwnd), GWL_EXSTYLE) or 0)
|
||
|
||
|
||
def iter_toplevel_hwnds_zorder():
|
||
"""Front-to-back top-level windows (generic Win32 Z-order walk)."""
|
||
hwnd = int(user32.GetTopWindow(None) or 0)
|
||
seen: set[int] = set()
|
||
while hwnd:
|
||
if hwnd in seen:
|
||
break
|
||
seen.add(hwnd)
|
||
yield hwnd
|
||
hwnd = int(user32.GetWindow(hwnd, GW_HWNDNEXT) or 0)
|
||
|
||
|
||
def list_capturable_candidates(
|
||
*,
|
||
exclude_hwnds: Optional[set[int]] = None,
|
||
min_width: int = 120,
|
||
min_height: int = 80,
|
||
) -> list[WindowInfo]:
|
||
"""Z-order capturable windows; no app-name hardcoding."""
|
||
exclude = {int(h) for h in (exclude_hwnds or set()) if h}
|
||
# Also exclude top-level of each excluded hwnd
|
||
exclude |= {get_toplevel_hwnd(h) for h in list(exclude)}
|
||
|
||
out: list[WindowInfo] = []
|
||
for hwnd in iter_toplevel_hwnds_zorder():
|
||
if hwnd in exclude:
|
||
continue
|
||
if not user32.IsWindowVisible(hwnd):
|
||
continue
|
||
if user32.IsIconic(hwnd):
|
||
continue
|
||
if _is_cloaked(hwnd):
|
||
continue
|
||
rect = _window_rect(hwnd)
|
||
if not rect or rect.get("width", 0) < min_width or rect.get("height", 0) < min_height:
|
||
continue
|
||
try:
|
||
info = window_info_from_hwnd(hwnd)
|
||
except Exception:
|
||
continue
|
||
if "UI Inspector" in (info.title or "") or "OpenClaw UI Inspector" in (info.title or ""):
|
||
continue
|
||
if is_system_chrome_window(info) and info.class_name in {
|
||
"Shell_TrayWnd",
|
||
"Shell_SecondaryTrayWnd",
|
||
"NotifyIconOverflowWindow",
|
||
"Progman",
|
||
"WorkerW",
|
||
}:
|
||
continue
|
||
if is_system_chrome_window(info) and not (info.title or "").strip():
|
||
continue
|
||
# Skip untitled toolwindows (IME, ghosts)
|
||
ex = _exstyle(hwnd)
|
||
if (ex & WS_EX_TOOLWINDOW) and not (info.title or "").strip():
|
||
continue
|
||
out.append(info)
|
||
return out
|
||
|
||
|
||
def _owner_root(hwnd: int) -> int:
|
||
cur = int(hwnd)
|
||
seen: set[int] = set()
|
||
while cur and cur not in seen:
|
||
seen.add(cur)
|
||
owner = int(user32.GetWindow(cur, GW_OWNER) or 0)
|
||
if not owner:
|
||
break
|
||
cur = owner
|
||
return cur or int(hwnd)
|
||
|
||
|
||
def prefer_last_active_popup(info: WindowInfo) -> WindowInfo:
|
||
"""If Win32 reports an active owned popup, prefer it (still app-agnostic)."""
|
||
try:
|
||
root = _owner_root(info.hwnd)
|
||
popup = int(user32.GetLastActivePopup(root) or 0)
|
||
if not popup or popup == int(info.hwnd):
|
||
# also try popup of the candidate itself
|
||
popup = int(user32.GetLastActivePopup(int(info.hwnd)) or 0)
|
||
if not popup or not user32.IsWindowVisible(popup) or user32.IsIconic(popup):
|
||
return info
|
||
if _is_cloaked(popup):
|
||
return info
|
||
prect = _window_rect(popup)
|
||
if not prect or prect.get("width", 0) < 80 or prect.get("height", 0) < 60:
|
||
return info
|
||
popup_info = window_info_from_hwnd(popup)
|
||
if is_system_chrome_window(popup_info):
|
||
return info
|
||
return popup_info
|
||
except Exception:
|
||
return info
|
||
|
||
|
||
def window_info_under_cursor(
|
||
*,
|
||
exclude_hwnds: Optional[set[int]] = None,
|
||
) -> Optional[WindowInfo]:
|
||
"""光标下的顶层窗（不 Activate、不点击）。光标在 Inspector 上则返回 None。"""
|
||
exclude = {int(h) for h in (exclude_hwnds or set()) if h}
|
||
exclude |= {get_toplevel_hwnd(h) for h in list(exclude)}
|
||
pt = wintypes.POINT()
|
||
if not user32.GetCursorPos(ctypes.byref(pt)):
|
||
return None
|
||
hwnd = int(user32.WindowFromPoint(pt) or 0)
|
||
if not hwnd:
|
||
return None
|
||
top = get_toplevel_hwnd(hwnd)
|
||
if not top or top in exclude:
|
||
return None
|
||
if not user32.IsWindowVisible(top) or user32.IsIconic(top) or _is_cloaked(top):
|
||
return None
|
||
try:
|
||
info = window_info_from_hwnd(top)
|
||
except Exception:
|
||
return None
|
||
if is_inspector_window(info):
|
||
return None
|
||
if "UI Inspector" in (info.title or "") or "OpenClaw UI Inspector" in (info.title or ""):
|
||
return None
|
||
return info
|
||
|
||
|
||
def find_wecom_search_result_window(
|
||
*,
|
||
pid: Optional[int] = None,
|
||
exclude_hwnds: Optional[set[int]] = None,
|
||
) -> Optional[WindowInfo]:
|
||
"""同进程可见的 SearchResultWindow2（搜索联系人浮层），不要求前台焦点。"""
|
||
exclude = {int(h) for h in (exclude_hwnds or set()) if h}
|
||
exclude |= {get_toplevel_hwnd(h) for h in list(exclude)}
|
||
best: Optional[WindowInfo] = None
|
||
best_area = -1
|
||
for hwnd in iter_toplevel_hwnds_zorder():
|
||
if hwnd in exclude:
|
||
continue
|
||
if not user32.IsWindowVisible(hwnd) or user32.IsIconic(hwnd) or _is_cloaked(hwnd):
|
||
continue
|
||
cls_buf = ctypes.create_unicode_buffer(256)
|
||
user32.GetClassNameW(hwnd, cls_buf, 256)
|
||
if (cls_buf.value or "") != "SearchResultWindow2":
|
||
continue
|
||
try:
|
||
info = window_info_from_hwnd(hwnd)
|
||
except Exception:
|
||
continue
|
||
if pid is not None and int(info.pid) != int(pid):
|
||
continue
|
||
rect = _window_rect(hwnd) or {}
|
||
area = int(rect.get("width") or 0) * int(rect.get("height") or 0)
|
||
if area < 40 * 40:
|
||
continue
|
||
if area > best_area:
|
||
best_area = area
|
||
best = info
|
||
return best
|
||
|
||
|
||
def foreground_toplevel_hwnd(*, exclude_hwnds: Optional[set[int]] = None) -> Optional[int]:
|
||
"""True foreground top-level hwnd (GetForegroundWindow), if capturable."""
|
||
exclude = {int(h) for h in (exclude_hwnds or set()) if h}
|
||
exclude |= {get_toplevel_hwnd(h) for h in list(exclude)}
|
||
try:
|
||
fg = int(user32.GetForegroundWindow() or 0)
|
||
except Exception:
|
||
fg = 0
|
||
if not fg:
|
||
return None
|
||
top = get_toplevel_hwnd(fg)
|
||
if top in exclude:
|
||
return None
|
||
if not user32.IsWindowVisible(top) or user32.IsIconic(top) or _is_cloaked(top):
|
||
return None
|
||
return int(top)
|
||
|
||
|
||
def find_capture_target(
|
||
*,
|
||
exclude_hwnds: Optional[set[int]] = None,
|
||
prefer_pid: Optional[int] = None,
|
||
under_cursor: bool = False,
|
||
prefer_wecom_search_popup: bool = True,
|
||
) -> tuple[WindowInfo, list[WindowInfo]]:
|
||
"""
|
||
Pick a capturable top-level window.
|
||
|
||
Default (under_cursor=False) keeps legacy behaviour:
|
||
1) GetForegroundWindow, if capturable
|
||
2) else Z-order top capturable candidate
|
||
3) prefer_last_active_popup (same pid)
|
||
|
||
Optional:
|
||
- under_cursor: 光标下顶层窗优先（不点击、不抢焦点）；失败则回退前台逻辑
|
||
- prefer_wecom_search_popup: 若候选为企微主窗且存在 SearchResultWindow2，改采浮层
|
||
"""
|
||
info, cands, _meta = resolve_capture_target(
|
||
exclude_hwnds=exclude_hwnds,
|
||
prefer_pid=prefer_pid,
|
||
under_cursor=under_cursor,
|
||
prefer_wecom_search_popup=prefer_wecom_search_popup,
|
||
)
|
||
return info, cands
|
||
|
||
|
||
def resolve_capture_target(
|
||
*,
|
||
exclude_hwnds: Optional[set[int]] = None,
|
||
prefer_pid: Optional[int] = None,
|
||
under_cursor: bool = False,
|
||
prefer_wecom_search_popup: bool = True,
|
||
) -> tuple[WindowInfo, list[WindowInfo], dict[str, Any]]:
|
||
"""与 find_capture_target 相同，额外返回 selection meta（strategy 链）。"""
|
||
candidates = list_capturable_candidates(exclude_hwnds=exclude_hwnds)
|
||
if not candidates:
|
||
raise RuntimeError(
|
||
"未找到可拾取的顶层窗口。请确认目标软件窗口可见且未被最小化。"
|
||
)
|
||
|
||
by_hwnd = {int(c.hwnd): c for c in candidates}
|
||
chosen: Optional[WindowInfo] = None
|
||
strategy_parts: list[str] = []
|
||
|
||
if under_cursor:
|
||
cur = window_info_under_cursor(exclude_hwnds=exclude_hwnds)
|
||
if cur is not None:
|
||
chosen = cur
|
||
strategy_parts.append("under_cursor")
|
||
|
||
if chosen is None:
|
||
fg = foreground_toplevel_hwnd(exclude_hwnds=exclude_hwnds)
|
||
if fg and fg in by_hwnd:
|
||
# Frontmost always wins — never replace with a larger/sticky other app.
|
||
chosen = by_hwnd[fg]
|
||
strategy_parts.append("foreground")
|
||
elif fg:
|
||
# Foreground exists but filtered from candidates — still try to describe it.
|
||
try:
|
||
info = window_info_from_hwnd(fg)
|
||
if not is_inspector_window(info) and not (
|
||
is_system_chrome_window(info) and not (info.title or "").strip()
|
||
):
|
||
chosen = info
|
||
strategy_parts.append("foreground_raw")
|
||
except Exception:
|
||
chosen = None
|
||
|
||
if chosen is None:
|
||
# No usable foreground: Z-order front. prefer_pid only as soft filter here.
|
||
if prefer_pid:
|
||
for c in candidates:
|
||
if int(c.pid) == int(prefer_pid):
|
||
chosen = c
|
||
strategy_parts.append("sticky_pid")
|
||
break
|
||
if chosen is None:
|
||
chosen = candidates[0]
|
||
strategy_parts.append("zorder")
|
||
|
||
# Owned popup of the *same* front window only (e.g. search panel), never another app
|
||
popup = prefer_last_active_popup(chosen)
|
||
if int(popup.pid) == int(chosen.pid) and int(popup.hwnd) != int(chosen.hwnd):
|
||
chosen = popup
|
||
strategy_parts.append("last_active_popup")
|
||
|
||
# 企微：主窗为前台时，若搜索浮层仍打开，优先采浮层（无需点击进会话）
|
||
if prefer_wecom_search_popup and (chosen.class_name or "") == "WeWorkWindow":
|
||
search_win = find_wecom_search_result_window(
|
||
pid=int(chosen.pid), exclude_hwnds=exclude_hwnds
|
||
)
|
||
if search_win is not None:
|
||
chosen = search_win
|
||
strategy_parts.append("wecom_search_popup")
|
||
|
||
meta = {
|
||
"strategy": "+".join(strategy_parts) if strategy_parts else "unknown",
|
||
"under_cursor": bool(under_cursor),
|
||
"prefer_wecom_search_popup": bool(prefer_wecom_search_popup),
|
||
"chosen_class": chosen.class_name,
|
||
"chosen_hwnd": int(chosen.hwnd),
|
||
}
|
||
return chosen, candidates, meta
|
||
|
||
|
||
def wait_frontmost_target(
|
||
*,
|
||
exclude_hwnds: Optional[set[int]] = None,
|
||
timeout_sec: float = 12.0,
|
||
poll_sec: float = 0.35,
|
||
) -> tuple[WindowInfo, list[WindowInfo], dict[str, Any]]:
|
||
"""
|
||
Wait until the true foreground window is a capturable target
|
||
(not the Inspector). Frontmost = the capture target. No size ranking.
|
||
"""
|
||
ensure_com_sta()
|
||
ensure_uia()
|
||
exclude = {int(h) for h in (exclude_hwnds or set()) if h}
|
||
deadline = time.time() + float(timeout_sec)
|
||
meta: dict[str, Any] = {"waited": False, "polls": 0}
|
||
last_err = ""
|
||
while time.time() < deadline:
|
||
meta["polls"] += 1
|
||
try:
|
||
info, cands = find_capture_target(exclude_hwnds=exclude_hwnds)
|
||
except Exception as exc:
|
||
last_err = str(exc)
|
||
time.sleep(poll_sec)
|
||
continue
|
||
# If foreground resolved to an excluded hwnd path, candidates[0] is next.
|
||
# Reject only when we somehow still landed on Inspector.
|
||
if is_inspector_window(info):
|
||
time.sleep(poll_sec)
|
||
meta["waited"] = True
|
||
continue
|
||
meta["window"] = info.to_dict()
|
||
return info, cands, meta
|
||
if last_err:
|
||
raise RuntimeError(f"等待最前端目标窗口超时：{last_err}")
|
||
raise RuntimeError(
|
||
"等待最前端目标窗口超时：请先用鼠标点一下要采集的软件窗口（不要点 Inspector）。"
|
||
)
|
||
|
||
|
||
def prepare_frontmost_for_capture(
|
||
*,
|
||
exclude_hwnds: Optional[set[int]] = None,
|
||
timeout_sec: float = 12.0,
|
||
) -> dict[str, Any]:
|
||
"""
|
||
Resolve frontmost target, light-activate if mainish (no force maximize), warmup.
|
||
Always returns the frontmost window — findall is informational only,
|
||
never a reason to pick another app.
|
||
|
||
企微：先静默调用 kit ``ensure_ui_ready("wecom")``（与技能同路径），再轻量 poke。
|
||
"""
|
||
info, cands, wait_meta = wait_frontmost_target(
|
||
exclude_hwnds=exclude_hwnds, timeout_sec=timeout_sec
|
||
)
|
||
act = None
|
||
if is_mainish_window(info):
|
||
# Keep operator size/position; only restore if minimized.
|
||
act = activate_main_hwnd(info.hwnd, maximize=False)
|
||
|
||
# Product adapters may need to make their accessibility provider ready
|
||
# before the operator starts typing. Running this on the app's existing
|
||
# startup worker keeps Tk responsive and prevents the first input
|
||
# transaction from racing a 10-20 second background warmup.
|
||
import kit_wake
|
||
|
||
kit_report = kit_wake.auto_ensure_if_wecom(
|
||
class_name=info.class_name or "",
|
||
title=info.title or "",
|
||
hwnd=info.hwnd,
|
||
)
|
||
findall = kit_report.get("findall") if kit_report.get("ok") else None
|
||
provider_cold = bool(kit_report.get("applicable") and not kit_report.get("ok"))
|
||
|
||
return {
|
||
"ready": True, # frontmost resolved => ready to capture that window
|
||
"window": info.to_dict(),
|
||
"candidates": len(cands),
|
||
"findall": findall,
|
||
"uia_sparse": bool(provider_cold),
|
||
"provider_cold": provider_cold,
|
||
"activate": act,
|
||
"warmup": {"mode": "startup_adapter" if kit_report.get("applicable") else "deferred"},
|
||
"kit_wake": kit_report,
|
||
"wait": wait_meta,
|
||
"strategy": "foreground_only",
|
||
}
|
||
|
||
|
||
def enum_child_hwnds(parent: int, *, max_count: int = 80) -> list[int]:
|
||
result: list[int] = []
|
||
|
||
@ctypes.WINFUNCTYPE(ctypes.c_bool, wintypes.HWND, wintypes.LPARAM)
|
||
def _cb(hwnd, _lparam):
|
||
if len(result) >= max_count:
|
||
return False
|
||
if user32.IsWindowVisible(hwnd):
|
||
result.append(int(hwnd))
|
||
return True
|
||
|
||
user32.EnumChildWindows(int(parent), _cb, 0)
|
||
return result
|
||
|
||
|
||
def is_window_minimized(hwnd: int) -> bool:
|
||
hwnd = int(hwnd)
|
||
if user32.IsIconic(hwnd):
|
||
return True
|
||
rect = _window_rect(hwnd)
|
||
# Windows parks minimized windows near (-32000,-32000)
|
||
if rect and int(rect.get("left") or 0) <= -30000 and int(rect.get("top") or 0) <= -30000:
|
||
return True
|
||
return False
|
||
|
||
|
||
def restore_window_if_needed(
|
||
hwnd: int,
|
||
*,
|
||
force_show: bool = False,
|
||
bring_foreground: bool = False,
|
||
) -> dict[str, Any]:
|
||
"""
|
||
Only restore when minimized by default.
|
||
|
||
Avoid unconditional ShowWindow/SetForeground — repeating those on child/tool
|
||
HWNDs (esp. WeCom) spawns/flickers many tiny windows under the process.
|
||
"""
|
||
hwnd = int(hwnd)
|
||
info = {
|
||
"hwnd": hwnd,
|
||
"was_minimized": False,
|
||
"restored": False,
|
||
"shown": False,
|
||
"foreground": False,
|
||
}
|
||
if is_window_minimized(hwnd):
|
||
info["was_minimized"] = True
|
||
user32.ShowWindow(hwnd, SW_RESTORE)
|
||
time.sleep(0.2)
|
||
info["restored"] = True
|
||
elif force_show:
|
||
try:
|
||
user32.ShowWindow(hwnd, SW_SHOW)
|
||
info["shown"] = True
|
||
except Exception:
|
||
pass
|
||
if bring_foreground:
|
||
try:
|
||
if user32.SetForegroundWindow(hwnd):
|
||
info["foreground"] = True
|
||
except Exception:
|
||
pass
|
||
time.sleep(0.05)
|
||
return info
|
||
|
||
|
||
def findall_count_for_hwnd(hwnd: int) -> int:
|
||
try:
|
||
from uiautomation.uiautomation import _AutomationClient
|
||
|
||
root = control_from_hwnd(int(hwnd))
|
||
ac = _AutomationClient.instance()
|
||
arr = root.Element.FindAll(7, ac.IUIAutomation.CreateTrueCondition())
|
||
return int(arr.Length)
|
||
except Exception:
|
||
return -1
|
||
|
||
|
||
def poke_uia_provider(hwnd: int) -> dict[str, Any]:
|
||
"""
|
||
Light provider poke: no ShowWindow / no SetForeground.
|
||
|
||
Safe to call on child or sibling HWNDs without popping tool windows.
|
||
"""
|
||
ensure_uia()
|
||
hwnd = int(hwnd)
|
||
report: dict[str, Any] = {"hwnd": hwnd, "mode": "light", "steps": []}
|
||
for obj_id, label in (
|
||
(UIA_ROOT_OBJECT_ID, "UiaRootObjectId"),
|
||
(OBJID_CLIENT, "OBJID_CLIENT"),
|
||
):
|
||
try:
|
||
res = int(user32.SendMessageW(hwnd, WM_GETOBJECT, 0, obj_id) or 0)
|
||
report["steps"].append({"wm_getobject": label, "lresult": res})
|
||
except Exception as exc:
|
||
report["steps"].append({"wm_getobject": label, "error": str(exc)})
|
||
try:
|
||
from uiautomation.uiautomation import _AutomationClient
|
||
import uiautomation as auto
|
||
|
||
ac = _AutomationClient.instance()
|
||
el = ac.IUIAutomation.ElementFromHandle(wintypes.HWND(hwnd))
|
||
report["steps"].append({"ElementFromHandle": el is not None})
|
||
ctrl = auto.ControlFromHandle(hwnd)
|
||
try:
|
||
arr = ctrl.Element.FindAll(4, ac.IUIAutomation.CreateTrueCondition())
|
||
report["steps"].append({"poke_FindAll_descendants": int(arr.Length)})
|
||
except Exception as exc:
|
||
report["steps"].append({"poke_FindAll": str(exc)})
|
||
except Exception as exc:
|
||
report["steps"].append({"uia_refresh": str(exc)})
|
||
return report
|
||
|
||
|
||
def _msaa_and_uia_poke(hwnd: int, report: dict[str, Any]) -> None:
|
||
"""Shared MSAA/UIA attach steps (no ShowWindow)."""
|
||
hwnd = int(hwnd)
|
||
for obj_id, label in (
|
||
(UIA_ROOT_OBJECT_ID, "UiaRootObjectId"),
|
||
(OBJID_CLIENT, "OBJID_CLIENT"),
|
||
(OBJID_WINDOW, "OBJID_WINDOW"),
|
||
):
|
||
try:
|
||
res = int(user32.SendMessageW(hwnd, WM_GETOBJECT, 0, obj_id) or 0)
|
||
report["steps"].append({"wm_getobject": label, "lresult": res})
|
||
except Exception as exc:
|
||
report["steps"].append({"wm_getobject": label, "error": str(exc)})
|
||
|
||
try:
|
||
from comtypes import GUID, IUnknown
|
||
|
||
iid = GUID("{618736e0-3c3d-11cf-810c-00aa00389b71}") # IID_IAccessible
|
||
punk = ctypes.c_void_p()
|
||
hr = ctypes.windll.oleacc.AccessibleObjectFromWindow(
|
||
wintypes.HWND(hwnd),
|
||
ctypes.c_long(OBJID_CLIENT),
|
||
ctypes.byref(iid),
|
||
ctypes.byref(punk),
|
||
)
|
||
report["steps"].append(
|
||
{
|
||
"AccessibleObjectFromWindow": int(hr),
|
||
"has_punk": bool(punk.value),
|
||
}
|
||
)
|
||
if punk.value:
|
||
try:
|
||
ctypes.cast(punk, ctypes.POINTER(IUnknown)).Release()
|
||
except Exception:
|
||
pass
|
||
except Exception as exc:
|
||
report["steps"].append({"AccessibleObjectFromWindow": str(exc)})
|
||
|
||
try:
|
||
from uiautomation.uiautomation import _AutomationClient
|
||
import uiautomation as auto
|
||
|
||
ac = _AutomationClient.instance()
|
||
el = ac.IUIAutomation.ElementFromHandle(wintypes.HWND(hwnd))
|
||
report["steps"].append({"ElementFromHandle": el is not None})
|
||
ctrl = auto.ControlFromHandle(hwnd)
|
||
try:
|
||
kids = ctrl.GetChildren()
|
||
report["steps"].append({"GetChildren_count": len(kids or [])})
|
||
except Exception as exc:
|
||
report["steps"].append({"GetChildren": str(exc)})
|
||
try:
|
||
arr = ctrl.Element.FindAll(4, ac.IUIAutomation.CreateTrueCondition())
|
||
report["steps"].append({"poke_FindAll_descendants": int(arr.Length)})
|
||
except Exception as exc:
|
||
report["steps"].append({"poke_FindAll": str(exc)})
|
||
except Exception as exc:
|
||
report["steps"].append({"uia_refresh": str(exc)})
|
||
|
||
|
||
def warmup_uia_provider(
|
||
hwnd: int,
|
||
*,
|
||
interact_window: bool = True,
|
||
allow_foreground_nudge: bool = True,
|
||
sparse_findall_threshold: int = 8,
|
||
) -> dict[str, Any]:
|
||
"""
|
||
Force accessibility provider initialization (Inspect-like).
|
||
|
||
- Restores minimized main window only (no blanket Show on children).
|
||
- If still sparse after poke, optionally SetForeground once on THIS hwnd
|
||
(never on child/tool HWNDs — that was the popup storm).
|
||
"""
|
||
ensure_uia()
|
||
hwnd = int(hwnd)
|
||
report: dict[str, Any] = {"hwnd": hwnd, "mode": "full", "steps": []}
|
||
|
||
if interact_window:
|
||
report["restore"] = restore_window_if_needed(
|
||
hwnd, force_show=False, bring_foreground=False
|
||
)
|
||
else:
|
||
report["restore"] = {"skipped": True}
|
||
|
||
report["findall_before"] = findall_count_for_hwnd(hwnd)
|
||
_msaa_and_uia_poke(hwnd, report)
|
||
time.sleep(0.15)
|
||
report["findall_after"] = findall_count_for_hwnd(hwnd)
|
||
|
||
if (
|
||
allow_foreground_nudge
|
||
and interact_window
|
||
and int(report.get("findall_after") or 0) < int(sparse_findall_threshold)
|
||
):
|
||
# One-shot focus nudge on the capture target only.
|
||
nudge = restore_window_if_needed(
|
||
hwnd, force_show=False, bring_foreground=True
|
||
)
|
||
report["foreground_nudge"] = nudge
|
||
time.sleep(0.35)
|
||
_msaa_and_uia_poke(hwnd, report)
|
||
report["findall_after_nudge"] = findall_count_for_hwnd(hwnd)
|
||
report["findall_after"] = report["findall_after_nudge"]
|
||
|
||
report["woke"] = int(report.get("findall_after") or 0) > max(
|
||
5, int(report.get("findall_before") or 0) + 2
|
||
)
|
||
report["provider_cold"] = is_provider_cold(
|
||
hwnd, findall=int(report.get("findall_after") or 0)
|
||
)
|
||
return report
|
||
|
||
|
||
def list_same_pid_toplevel_hwnds(pid: int) -> list[int]:
|
||
found: list[int] = []
|
||
|
||
@ctypes.WINFUNCTYPE(ctypes.c_bool, wintypes.HWND, wintypes.LPARAM)
|
||
def _cb(hwnd, _lp):
|
||
p = ctypes.c_ulong(0)
|
||
user32.GetWindowThreadProcessId(hwnd, ctypes.byref(p))
|
||
if int(p.value) == int(pid):
|
||
found.append(int(hwnd))
|
||
return True
|
||
|
||
user32.EnumWindows(_cb, 0)
|
||
return found
|
||
|
||
|
||
def dump_window_tree(
|
||
hwnd: int,
|
||
*,
|
||
max_depth: int = 16,
|
||
max_nodes: int = 8000,
|
||
only_visible: bool = False,
|
||
enrich_child_hwnds: bool = True,
|
||
use_findall: bool = True,
|
||
warmup: bool = True,
|
||
warmup_mode: str = "full",
|
||
merge_same_pid_if_sparse: bool = True,
|
||
) -> tuple[dict[str, Any], list[dict[str, Any]], dict[str, Any], list[dict[str, Any]]]:
|
||
"""
|
||
Dump UIA tree + flat index + locatable candidates.
|
||
|
||
warmup_mode:
|
||
- full: MSAA/UIA poke on target; restore only if minimized
|
||
- light: ElementFromHandle poke only (no ShowWindow)
|
||
- probe: light poke + FindAll, but never invoke a blocking product wake
|
||
- off: no provider warmup
|
||
Child / same-PID merges never call ShowWindow (light poke at most).
|
||
"""
|
||
import kit_wake
|
||
|
||
hwnd = int(hwnd)
|
||
mode = (warmup_mode or "full").lower()
|
||
if not warmup:
|
||
mode = "off"
|
||
if mode not in {"full", "light", "probe", "off"}:
|
||
mode = "full"
|
||
|
||
kit_report: dict[str, Any] = {"applicable": False, "skipped": True, "reason": "not_checked"}
|
||
try:
|
||
winfo_for_kit = window_info_from_hwnd(hwnd)
|
||
except Exception:
|
||
winfo_for_kit = None
|
||
# 搜索浮层稀疏时勿合并主窗会话列表，否则验收会误判「已有数据」
|
||
allow_same_pid_merge = bool(merge_same_pid_if_sparse)
|
||
if winfo_for_kit is not None and (winfo_for_kit.class_name or "") == "SearchResultWindow2":
|
||
allow_same_pid_merge = False
|
||
if winfo_for_kit is not None and mode not in {"off", "probe"}:
|
||
hint = findall_count_for_hwnd(hwnd)
|
||
kit_report = kit_wake.auto_ensure_if_wecom(
|
||
class_name=winfo_for_kit.class_name or "",
|
||
title=winfo_for_kit.title or "",
|
||
hwnd=hwnd,
|
||
force=False,
|
||
findall_hint=hint,
|
||
)
|
||
|
||
warmup_report: dict[str, Any] = {"mode": mode}
|
||
if mode == "full":
|
||
try:
|
||
winfo = window_info_from_hwnd(hwnd)
|
||
except Exception:
|
||
winfo = None
|
||
if winfo is not None and is_mainish_window(winfo):
|
||
# No force maximize — preserve operator window geometry.
|
||
act = activate_main_hwnd(hwnd, maximize=False)
|
||
warmup_report = warmup_uia_provider(
|
||
hwnd, interact_window=False, allow_foreground_nudge=False
|
||
)
|
||
warmup_report["activate"] = act
|
||
else:
|
||
warmup_report = warmup_uia_provider(
|
||
hwnd, interact_window=True, allow_foreground_nudge=True
|
||
)
|
||
elif mode in {"light", "probe"}:
|
||
warmup_report = poke_uia_provider(hwnd)
|
||
|
||
def _run_dump() -> tuple[dict[str, Any], list[dict[str, Any]], dict[str, Any]]:
|
||
root_ctrl = control_from_hwnd(hwnd)
|
||
t, idx, st = dump_tree(
|
||
root_ctrl,
|
||
max_depth=max_depth,
|
||
max_nodes=max_nodes,
|
||
only_visible=only_visible,
|
||
)
|
||
st = dict(st)
|
||
st["enriched_child_hwnds"] = False
|
||
st["findall_count"] = 0
|
||
st["merged_same_pid_hwnds"] = []
|
||
|
||
if enrich_child_hwnds:
|
||
children_nodes: list[dict[str, Any]] = list(t.get("children") or [])
|
||
seen_hwnd = {int(hwnd)}
|
||
for child_hwnd in enum_child_hwnds(hwnd, max_count=80):
|
||
if child_hwnd in seen_hwnd:
|
||
continue
|
||
seen_hwnd.add(child_hwnd)
|
||
if int(st.get("nodes_kept") or 0) >= max_nodes:
|
||
st["truncated"] = True
|
||
break
|
||
try:
|
||
# Never full-warmup / ShowWindow child HWNDs.
|
||
child_root = control_from_hwnd(child_hwnd)
|
||
except Exception:
|
||
continue
|
||
remain = max_nodes - int(st.get("nodes_kept") or 0)
|
||
sub_tree, sub_index, sub_stats = dump_tree(
|
||
child_root,
|
||
max_depth=max(2, max_depth - 1),
|
||
max_nodes=max(1, remain),
|
||
only_visible=only_visible,
|
||
)
|
||
if int(sub_stats.get("nodes_kept") or 0) <= 1 and not (
|
||
(sub_tree.get("name") or "") or (sub_tree.get("value") or "")
|
||
):
|
||
if int(sub_stats.get("nodes_kept") or 0) <= 0:
|
||
continue
|
||
children_nodes.append(sub_tree)
|
||
idx.extend(sub_index)
|
||
st["nodes_kept"] = int(st.get("nodes_kept") or 0) + int(
|
||
sub_stats.get("nodes_kept") or 0
|
||
)
|
||
st["nodes_visited"] = int(st.get("nodes_visited") or 0) + int(
|
||
sub_stats.get("nodes_visited") or 0
|
||
)
|
||
st["enriched_child_hwnds"] = True
|
||
if sub_stats.get("truncated"):
|
||
st["truncated"] = True
|
||
if children_nodes:
|
||
t = dict(t)
|
||
t["children"] = children_nodes
|
||
|
||
if use_findall and mode != "off":
|
||
found = collect_findall_flat(root_ctrl, max_nodes=max_nodes)
|
||
st["findall_count"] = len(found)
|
||
existing = {r.get("path_hint") for r in idx}
|
||
for row in found:
|
||
if row.get("path_hint") in existing:
|
||
continue
|
||
idx.append(row)
|
||
existing.add(row.get("path_hint"))
|
||
return t, idx, st
|
||
|
||
tree, index, stats = _run_dump()
|
||
if winfo_for_kit is not None:
|
||
kit_wake.note_provider_warm(
|
||
class_name=winfo_for_kit.class_name or "",
|
||
title=winfo_for_kit.title or "",
|
||
hwnd=hwnd,
|
||
findall=int(stats.get("findall_count") or 0),
|
||
)
|
||
stats["warmup"] = warmup_report
|
||
stats["warmup_mode"] = mode
|
||
stats["kit_wake"] = kit_report
|
||
stats["walker"] = "GetFirstChild/NextSibling (package default RawViewWalker)"
|
||
stats["escalated_warmup"] = False
|
||
|
||
locatable = locatable_from_index(index)
|
||
stats["locatable_count"] = len(locatable)
|
||
stats["uia_sparse"] = int(stats.get("locatable_count") or 0) < 8
|
||
stats["provider_cold"] = is_provider_cold(
|
||
hwnd, findall=int(stats.get("findall_count") or 0)
|
||
)
|
||
|
||
# 仍冷且目标为企微：强制再 ensure 一次（绕过短时 cache）后重 dump
|
||
if mode not in {"off", "probe"} and stats["provider_cold"] and winfo_for_kit is not None and kit_wake.looks_like_wecom(
|
||
class_name=winfo_for_kit.class_name or "",
|
||
title=winfo_for_kit.title or "",
|
||
):
|
||
kit_report = kit_wake.auto_ensure_if_wecom(
|
||
class_name=winfo_for_kit.class_name or "",
|
||
title=winfo_for_kit.title or "",
|
||
hwnd=hwnd,
|
||
force=True,
|
||
)
|
||
stats["kit_wake"] = kit_report
|
||
if kit_report.get("ok"):
|
||
tree, index, stats = _run_dump()
|
||
stats["warmup"] = warmup_report
|
||
stats["warmup_mode"] = mode
|
||
stats["kit_wake"] = kit_report
|
||
stats["walker"] = "GetFirstChild/NextSibling (package default RawViewWalker)"
|
||
stats["escalated_warmup"] = False
|
||
stats["kit_wake_retry"] = True
|
||
locatable = locatable_from_index(index)
|
||
stats["locatable_count"] = len(locatable)
|
||
stats["uia_sparse"] = int(stats.get("locatable_count") or 0) < 8
|
||
stats["provider_cold"] = is_provider_cold(
|
||
hwnd, findall=int(stats.get("findall_count") or 0)
|
||
)
|
||
|
||
# If light dump is sparse, escalate once to full warmup + re-dump (no child ShowWindow).
|
||
if stats["uia_sparse"] and mode == "light":
|
||
try:
|
||
winfo = window_info_from_hwnd(hwnd)
|
||
except Exception:
|
||
winfo = None
|
||
if winfo is not None and is_mainish_window(winfo):
|
||
act = activate_main_hwnd(hwnd, maximize=False)
|
||
warmup_report = warmup_uia_provider(
|
||
hwnd, interact_window=False, allow_foreground_nudge=False
|
||
)
|
||
warmup_report["activate"] = act
|
||
else:
|
||
warmup_report = warmup_uia_provider(
|
||
hwnd, interact_window=True, allow_foreground_nudge=True
|
||
)
|
||
tree, index, stats = _run_dump()
|
||
stats["warmup"] = warmup_report
|
||
stats["warmup_mode"] = "full"
|
||
stats["kit_wake"] = kit_report
|
||
stats["escalated_warmup"] = True
|
||
locatable = locatable_from_index(index)
|
||
stats["locatable_count"] = len(locatable)
|
||
stats["uia_sparse"] = int(stats.get("locatable_count") or 0) < 8
|
||
stats["provider_cold"] = is_provider_cold(
|
||
hwnd, findall=int(stats.get("findall_count") or 0)
|
||
)
|
||
|
||
# Sparse fallback: light poke + FindAll on a few large same-pid top-levels.
|
||
# Includes WeWorkWindow when current target is a thin popup.
|
||
# Never merge into SearchResultWindow2 (see allow_same_pid_merge).
|
||
stats["merge_same_pid_allowed"] = allow_same_pid_merge
|
||
if allow_same_pid_merge and stats["uia_sparse"]:
|
||
try:
|
||
info = window_info_from_hwnd(hwnd)
|
||
merged_n = 0
|
||
# Prefer large main-ish windows first
|
||
others: list[tuple[int, int, Any]] = []
|
||
for other in list_same_pid_toplevel_hwnds(info.pid):
|
||
if other == hwnd:
|
||
continue
|
||
rect = _window_rect(other)
|
||
if not rect:
|
||
continue
|
||
area = int(rect.get("width") or 0) * int(rect.get("height") or 0)
|
||
if int(rect.get("width") or 0) < 120 or int(rect.get("height") or 0) < 80:
|
||
continue
|
||
others.append((area, other, rect))
|
||
others.sort(key=lambda x: -x[0])
|
||
for _area, other, _rect in others:
|
||
if merged_n >= 8:
|
||
break
|
||
try:
|
||
oinfo = window_info_from_hwnd(other)
|
||
except Exception:
|
||
continue
|
||
if is_inspector_window(oinfo):
|
||
continue
|
||
if oinfo.class_name in {
|
||
"Shell_TrayWnd",
|
||
"Shell_SecondaryTrayWnd",
|
||
"Progman",
|
||
"WorkerW",
|
||
"IME",
|
||
"MSCTFIME UI",
|
||
"tooltips_class32",
|
||
}:
|
||
continue
|
||
# Light poke only — never ShowWindow on siblings
|
||
poke_uia_provider(other)
|
||
try:
|
||
oroot = control_from_hwnd(other)
|
||
extra = collect_findall_flat(
|
||
oroot, max_nodes=max(100, max_nodes - len(index))
|
||
)
|
||
except Exception:
|
||
continue
|
||
if len(extra) < 5:
|
||
continue
|
||
merged_n += 1
|
||
stats.setdefault("merged_same_pid_hwnds", []).append(
|
||
{
|
||
"hwnd": other,
|
||
"title": oinfo.title,
|
||
"class_name": oinfo.class_name,
|
||
"findall": len(extra),
|
||
}
|
||
)
|
||
existing = {r.get("path_hint") for r in index}
|
||
for row in extra:
|
||
row = dict(row)
|
||
row["source"] = f"findall_pid:{other}"
|
||
if row.get("path_hint") in existing:
|
||
continue
|
||
index.append(row)
|
||
existing.add(row.get("path_hint"))
|
||
locatable = locatable_from_index(index)
|
||
stats["locatable_count"] = len(locatable)
|
||
stats["findall_count"] = max(
|
||
int(stats.get("findall_count") or 0),
|
||
sum(1 for r in index if str(r.get("source", "")).startswith("findall")),
|
||
)
|
||
stats["uia_sparse"] = int(stats.get("locatable_count") or 0) < 8
|
||
except Exception as exc:
|
||
stats["same_pid_merge_error"] = str(exc)
|
||
|
||
stats["kit_wake"] = kit_report
|
||
return tree, index, stats, locatable
|
||
|
||
|
||
def collect_findall_flat(root_ctrl, *, max_nodes: int = 8000) -> list[dict[str, Any]]:
|
||
"""COM FindAll(Subtree) — often finds nodes GetChildren walk misses."""
|
||
out: list[dict[str, Any]] = []
|
||
try:
|
||
from uiautomation.uiautomation import Control, _AutomationClient
|
||
|
||
ac = _AutomationClient.instance()
|
||
cond = ac.IUIAutomation.CreateTrueCondition()
|
||
# TreeScope_Subtree = Element|Children|Descendants = 7
|
||
arr = root_ctrl.Element.FindAll(7, cond)
|
||
length = int(arr.Length)
|
||
for i in range(min(length, max_nodes)):
|
||
try:
|
||
ele = arr.GetElement(i)
|
||
ctrl = Control.CreateControlFromElement(ele)
|
||
if ctrl is None:
|
||
continue
|
||
info = describe_control(ctrl, depth=-1, parent_chain=[])
|
||
out.append(
|
||
{
|
||
"depth": -1,
|
||
"source": "findall",
|
||
"name": info.name,
|
||
"value": info.value,
|
||
"automation_id": info.automation_id,
|
||
"control_type": info.control_type,
|
||
"class_name": info.class_name,
|
||
"is_enabled": info.is_enabled,
|
||
"is_offscreen": info.is_offscreen,
|
||
"rect": info.rect,
|
||
"path_hint": info.path_hint,
|
||
}
|
||
)
|
||
except Exception:
|
||
continue
|
||
except Exception:
|
||
return out
|
||
return out
|
||
|
||
|
||
def locatable_from_index(
|
||
index: list[dict[str, Any]], *, limit: int = 300
|
||
) -> list[dict[str, Any]]:
|
||
"""Nodes useful for locator drafting: non-empty Name / Value / AutomationId."""
|
||
rows: list[dict[str, Any]] = []
|
||
seen: set[str] = set()
|
||
for r in index:
|
||
name = (r.get("name") or "").strip()
|
||
value = (r.get("value") or "").strip()
|
||
aid = (r.get("automation_id") or "").strip()
|
||
if not (name or value or aid):
|
||
continue
|
||
# Skip ultra-generic noise alone
|
||
if name in {"", "pane", "Pane"} and not value and not aid:
|
||
continue
|
||
key = f"{r.get('control_type')}|{name}|{value}|{aid}"
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
rows.append(r)
|
||
if len(rows) >= limit:
|
||
break
|
||
# Prefer rows with value or automation_id
|
||
rows.sort(
|
||
key=lambda x: (
|
||
0 if (x.get("value") or x.get("automation_id")) else 1,
|
||
0 if x.get("name") else 1,
|
||
int(x.get("depth") if x.get("depth") is not None and x.get("depth") >= 0 else 99),
|
||
)
|
||
)
|
||
return rows
|
||
|
||
|
||
def format_locatable_text(locatable: list[dict[str, Any]], *, limit: int = 120) -> str:
|
||
lines = [
|
||
f"可定位候选（Name/Value/AutomationId 非空，共 {len(locatable)}，显示前 {min(limit, len(locatable))}）",
|
||
"",
|
||
]
|
||
for i, r in enumerate(locatable[:limit]):
|
||
lines.append(f"[{i}] {r.get('control_type') or '?'}")
|
||
if r.get("name"):
|
||
lines.append(f" Name: {r.get('name')}")
|
||
if r.get("value"):
|
||
lines.append(f" Value: {r.get('value')}")
|
||
if r.get("automation_id"):
|
||
lines.append(f" AutomationId: {r.get('automation_id')}")
|
||
if r.get("class_name"):
|
||
lines.append(f" Class: {r.get('class_name')}")
|
||
lines.append(
|
||
f" offscreen={r.get('is_offscreen')} enabled={r.get('is_enabled')} "
|
||
f"src={r.get('source') or 'tree'}"
|
||
)
|
||
lines.append("")
|
||
if not locatable:
|
||
lines.append("（无）UIA 可能过稀；请看截图目录中的 screen.png，或换页后再拾取。")
|
||
return "\n".join(lines)
|
||
|
||
|
||
|
||
def format_tree_text(tree: dict[str, Any], *, max_lines: int = 2000) -> str:
|
||
"""Human-readable indented tree for the panel."""
|
||
lines: list[str] = []
|
||
|
||
def walk(node: dict[str, Any], depth: int) -> None:
|
||
if len(lines) >= max_lines:
|
||
return
|
||
indent = " " * depth
|
||
ctype = node.get("control_type") or "?"
|
||
name = node.get("name") or ""
|
||
value = node.get("value") or ""
|
||
aid = node.get("automation_id") or ""
|
||
bits = [ctype]
|
||
if name:
|
||
bits.append(f"Name={name!r}")
|
||
if value:
|
||
bits.append(f"Value={value!r}")
|
||
if aid:
|
||
bits.append(f"AutomationId={aid!r}")
|
||
lines.append(indent + " | ".join(bits))
|
||
for child in node.get("children") or []:
|
||
if len(lines) >= max_lines:
|
||
lines.append(indent + " …（已截断显示）")
|
||
return
|
||
walk(child, depth + 1)
|
||
|
||
if tree:
|
||
walk(tree, 0)
|
||
else:
|
||
lines.append("(空树)")
|
||
if len(lines) >= max_lines:
|
||
lines.append(f"… 显示已截断到 {max_lines} 行；完整内容见 tree.json / index.jsonl")
|
||
return "\n".join(lines)
|
||
|
||
|
||
# Back-compat: older callers expected a single WindowInfo (foreground only).
|
||
# Do NOT name this resolve_capture_target — that name is the full 3-tuple API above.
|
||
def resolve_frontmost_window(
|
||
*,
|
||
self_hwnd: int = 0,
|
||
poll_sec: float = 0.15,
|
||
wait_sec: float = 2.0,
|
||
) -> WindowInfo:
|
||
_ = poll_sec, wait_sec
|
||
info, _cands = find_capture_target(
|
||
exclude_hwnds={int(self_hwnd)} if self_hwnd else None,
|
||
under_cursor=False,
|
||
prefer_wecom_search_popup=False,
|
||
)
|
||
return info
|
||
|
||
|
||
def hwnd_from_control(ctrl) -> int:
|
||
try:
|
||
h = int(getattr(ctrl, "NativeWindowHandle", 0) or 0)
|
||
if h:
|
||
return h
|
||
except Exception:
|
||
pass
|
||
# Walk parents for a native handle
|
||
cur = ctrl
|
||
for _ in range(24):
|
||
try:
|
||
h = int(getattr(cur, "NativeWindowHandle", 0) or 0)
|
||
if h:
|
||
return h
|
||
except Exception:
|
||
pass
|
||
try:
|
||
parent = cur.GetParentControl()
|
||
except Exception:
|
||
break
|
||
if parent is None:
|
||
break
|
||
cur = parent
|
||
return 0
|
||
|
||
|
||
def attach_window_by_click(*, timeout_sec: float = 60.0) -> WindowInfo:
|
||
"""Minimize race: user clicks anywhere on the target window to attach it."""
|
||
x, y = wait_for_left_click(timeout_sec=timeout_sec)
|
||
ctrl = control_from_point(x, y)
|
||
hwnd = hwnd_from_control(ctrl)
|
||
if not hwnd:
|
||
# Fallback: window from point via Win32
|
||
hwnd = int(user32.WindowFromPoint(wintypes.POINT(x, y)) or 0)
|
||
if not hwnd:
|
||
raise RuntimeError(f"点击位置 ({x},{y}) 无法解析到窗口")
|
||
info = window_info_from_hwnd(hwnd)
|
||
if "UI Inspector" in (info.title or ""):
|
||
raise RuntimeError("点到了 Inspector 自己，请点目标软件窗口（如企业微信/添加客户）。")
|
||
return info
|
||
|
||
|
||
def control_from_hwnd(hwnd: int):
|
||
ensure_uia()
|
||
import uiautomation as auto
|
||
|
||
ctrl = auto.ControlFromHandle(int(hwnd))
|
||
if ctrl is None:
|
||
raise RuntimeError(f"无法从 HWND={hwnd} 取得 UIA 根控件")
|
||
return ctrl
|
||
|
||
|
||
def control_from_point(x: int, y: int):
|
||
ensure_uia()
|
||
import uiautomation as auto
|
||
|
||
ctrl = auto.ControlFromPoint(int(x), int(y))
|
||
if ctrl is None:
|
||
raise RuntimeError(f"坐标 ({x},{y}) 处没有 UIA 控件")
|
||
return ctrl
|
||
|
||
|
||
def build_parent_chain(ctrl, *, max_ancestors: int = 12) -> list[dict[str, str]]:
|
||
chain: list[dict[str, str]] = []
|
||
cur = ctrl
|
||
for _ in range(max_ancestors):
|
||
try:
|
||
parent = cur.GetParentControl()
|
||
except Exception:
|
||
break
|
||
if parent is None:
|
||
break
|
||
chain.append(
|
||
{
|
||
"name": _safe_str(getattr(parent, "Name", None)),
|
||
"value": _read_value(parent),
|
||
"automation_id": _safe_str(getattr(parent, "AutomationId", None)),
|
||
"control_type": _control_type_name(parent),
|
||
"class_name": _safe_str(getattr(parent, "ClassName", None)),
|
||
}
|
||
)
|
||
cur = parent
|
||
chain.reverse()
|
||
return chain
|
||
|
||
|
||
def dump_tree(
|
||
root,
|
||
*,
|
||
max_depth: int = 16,
|
||
max_nodes: int = 8000,
|
||
only_visible: bool = False,
|
||
) -> tuple[dict[str, Any], list[dict[str, Any]], dict[str, Any]]:
|
||
"""Return (tree, flat_index, stats).
|
||
|
||
Important: even when a node is offscreen / zero-sized, we still recurse into
|
||
its children. Pruning the whole subtree was the main cause of empty dumps.
|
||
"""
|
||
stats = {
|
||
"nodes_visited": 0,
|
||
"nodes_kept": 0,
|
||
"nodes_skipped_self": 0,
|
||
"truncated": False,
|
||
"max_depth": max_depth,
|
||
"max_nodes": max_nodes,
|
||
"only_visible": only_visible,
|
||
}
|
||
flat: list[dict[str, Any]] = []
|
||
|
||
def walk(ctrl, depth: int, ancestors: list[dict[str, str]]) -> Optional[dict[str, Any]]:
|
||
if stats["nodes_kept"] >= max_nodes:
|
||
stats["truncated"] = True
|
||
return None
|
||
stats["nodes_visited"] += 1
|
||
|
||
info = describe_control(ctrl, depth=depth, parent_chain=ancestors)
|
||
skip_self = False
|
||
if only_visible and depth > 0:
|
||
rect = info.rect
|
||
if info.is_offscreen is True:
|
||
skip_self = True
|
||
if rect and rect.get("width", 0) <= 0 and rect.get("height", 0) <= 0:
|
||
skip_self = True
|
||
|
||
child_ancestors = ancestors + [
|
||
{
|
||
"name": info.name,
|
||
"value": info.value,
|
||
"automation_id": info.automation_id,
|
||
"control_type": info.control_type,
|
||
"class_name": info.class_name,
|
||
}
|
||
]
|
||
children_out: list[dict[str, Any]] = []
|
||
if depth < max_depth and not stats["truncated"]:
|
||
try:
|
||
children = ctrl.GetChildren()
|
||
except Exception:
|
||
children = []
|
||
for child in children or []:
|
||
if stats["truncated"] or stats["nodes_kept"] >= max_nodes:
|
||
stats["truncated"] = True
|
||
break
|
||
child_node = walk(child, depth + 1, child_ancestors)
|
||
if child_node is not None:
|
||
children_out.append(child_node)
|
||
|
||
if skip_self:
|
||
stats["nodes_skipped_self"] += 1
|
||
# Pass children up so hierarchy is not lost
|
||
if len(children_out) == 1:
|
||
return children_out[0]
|
||
if children_out:
|
||
return {
|
||
"control_type": info.control_type or "SkippedContainer",
|
||
"name": info.name,
|
||
"value": info.value,
|
||
"automation_id": info.automation_id,
|
||
"class_name": info.class_name,
|
||
"skipped_self": True,
|
||
"children": children_out,
|
||
}
|
||
return None
|
||
|
||
stats["nodes_kept"] += 1
|
||
node = info.to_dict()
|
||
flat.append(
|
||
{
|
||
"depth": depth,
|
||
"source": "tree",
|
||
"name": info.name,
|
||
"value": info.value,
|
||
"automation_id": info.automation_id,
|
||
"control_type": info.control_type,
|
||
"class_name": info.class_name,
|
||
"is_enabled": info.is_enabled,
|
||
"is_offscreen": info.is_offscreen,
|
||
"rect": info.rect,
|
||
"path_hint": info.path_hint,
|
||
}
|
||
)
|
||
if children_out:
|
||
node["children"] = children_out
|
||
return node
|
||
|
||
tree = walk(root, 0, []) or {}
|
||
return tree, flat, stats
|
||
|
||
|
||
def wait_for_left_click(
|
||
*,
|
||
timeout_sec: float = 60.0,
|
||
poll_sec: float = 0.05,
|
||
) -> tuple[int, int]:
|
||
"""Block until left mouse button is pressed then released; return cursor pos."""
|
||
VK_LBUTTON = 0x01
|
||
|
||
def down() -> bool:
|
||
return bool(user32.GetAsyncKeyState(VK_LBUTTON) & 0x8000)
|
||
|
||
deadline = time.time() + timeout_sec
|
||
# Wait until button is up first (avoid consuming the GUI click that started pick)
|
||
while time.time() < deadline and down():
|
||
time.sleep(poll_sec)
|
||
|
||
while time.time() < deadline:
|
||
if down():
|
||
while time.time() < deadline and down():
|
||
time.sleep(poll_sec)
|
||
point = wintypes.POINT()
|
||
user32.GetCursorPos(ctypes.byref(point))
|
||
return int(point.x), int(point.y)
|
||
time.sleep(poll_sec)
|
||
raise TimeoutError(f"等待鼠标左键点击超时（{timeout_sec:.0f}s）")
|
||
|
||
|
||
def pick_control_at_cursor_after_click(
|
||
*,
|
||
timeout_sec: float = 60.0,
|
||
attached_hwnd: Optional[int] = None,
|
||
) -> tuple[ControlInfo, dict[str, int]]:
|
||
x, y = wait_for_left_click(timeout_sec=timeout_sec)
|
||
ctrl = control_from_point(x, y)
|
||
info_hwnd = hwnd_from_control(ctrl) or int(
|
||
user32.WindowFromPoint(wintypes.POINT(int(x), int(y))) or 0
|
||
)
|
||
top = get_toplevel_hwnd(info_hwnd) if info_hwnd else 0
|
||
|
||
# Taskbar / desktop chrome — almost never what you want for skill RPA
|
||
class_buf = ctypes.create_unicode_buffer(256)
|
||
if top:
|
||
user32.GetClassNameW(top, class_buf, 256)
|
||
top_class = class_buf.value or ""
|
||
if top_class in {"Shell_TrayWnd", "Shell_SecondaryTrayWnd", "NotifyIconOverflowWindow"}:
|
||
raise RuntimeError(
|
||
f"点到了任务栏/系统托盘（class={top_class}），不是业务窗口。\n"
|
||
f"请点已附着窗口内部的控件，不要点任务栏上的 Python/Cursor 图标。"
|
||
)
|
||
|
||
if attached_hwnd:
|
||
attached_top = get_toplevel_hwnd(int(attached_hwnd))
|
||
if top and attached_top and top != attached_top:
|
||
title_buf = ctypes.create_unicode_buffer(512)
|
||
user32.GetWindowTextW(top, title_buf, 512)
|
||
raise RuntimeError(
|
||
"拾取点不在已附着窗口内。\n"
|
||
f"已附着 hwnd={attached_top}；实际点到 title={title_buf.value!r} "
|
||
f"class={top_class!r} hwnd={top}。\n"
|
||
"请重新「点选附着」到目标软件（如企业微信），再在该窗口内拾取。"
|
||
)
|
||
|
||
chain = build_parent_chain(ctrl)
|
||
info = describe_control(ctrl, depth=0, parent_chain=chain)
|
||
return info, {"x": x, "y": y}
|