client-commons/skill-template
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: archive tools/ui_inspector snapshot before migration removal.

Browse Source 
Keep the last in-template copy on main so history can restore it; product continues in jiangchang-ui-recorder.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
chendelian
2026-08-07 11:19:14 +08:00
parent 46bc573777
commit 633ecff4e2
92 changed files with 13704 additions and 288 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
tools/ui_inspector/desktop_uia.py
View File

@@ -508,6 +508,70 @@ def prefer_last_active_popup(info: WindowInfo) -> WindowInfo:
return info
def window_info_under_cursor(
*,
exclude_hwnds: Optional[set[int]] = None,
) -> Optional[WindowInfo]:
"""光标下的顶层窗（不 Activate、不点击）。光标在 Inspector 上则返回 None。"""
exclude = {int(h) for h in (exclude_hwnds or set()) if h}
exclude |= {get_toplevel_hwnd(h) for h in list(exclude)}
pt = wintypes.POINT()
if not user32.GetCursorPos(ctypes.byref(pt)):
return None
hwnd = int(user32.WindowFromPoint(pt) or 0)
if not hwnd:
return None
top = get_toplevel_hwnd(hwnd)
if not top or top in exclude:
return None
if not user32.IsWindowVisible(top) or user32.IsIconic(top) or _is_cloaked(top):
return None
try:
info = window_info_from_hwnd(top)
except Exception:
return None
if is_inspector_window(info):
return None
if "UI Inspector" in (info.title or "") or "OpenClaw UI Inspector" in (info.title or ""):
return None
return info
def find_wecom_search_result_window(
*,
pid: Optional[int] = None,
exclude_hwnds: Optional[set[int]] = None,
) -> Optional[WindowInfo]:
"""同进程可见的 SearchResultWindow2（搜索联系人浮层），不要求前台焦点。"""
exclude = {int(h) for h in (exclude_hwnds or set()) if h}
exclude |= {get_toplevel_hwnd(h) for h in list(exclude)}
best: Optional[WindowInfo] = None
best_area = -1
for hwnd in iter_toplevel_hwnds_zorder():
if hwnd in exclude:
continue
if not user32.IsWindowVisible(hwnd) or user32.IsIconic(hwnd) or _is_cloaked(hwnd):
continue
cls_buf = ctypes.create_unicode_buffer(256)
user32.GetClassNameW(hwnd, cls_buf, 256)
if (cls_buf.value or "") != "SearchResultWindow2":
continue
try:
info = window_info_from_hwnd(hwnd)
except Exception:
continue
if pid is not None and int(info.pid) != int(pid):
continue
rect = _window_rect(hwnd) or {}
area = int(rect.get("width") or 0) * int(rect.get("height") or 0)
if area < 40 * 40:
continue
if area > best_area:
best_area = area
best = info
return best
def foreground_toplevel_hwnd(*, exclude_hwnds: Optional[set[int]] = None) -> Optional[int]:
"""True foreground top-level hwnd (GetForegroundWindow), if capturable."""
exclude = {int(h) for h in (exclude_hwnds or set()) if h}
@@ -530,16 +594,38 @@ def find_capture_target(
*,
exclude_hwnds: Optional[set[int]] = None,
prefer_pid: Optional[int] = None,
under_cursor: bool = False,
prefer_wecom_search_popup: bool = True,
) -> tuple[WindowInfo, list[WindowInfo]]:
"""
Pick the frontmost capturable window.
Pick a capturable top-level window.
Priority:
1) GetForegroundWindow (true frontmost), if capturable
Default (under_cursor=False) keeps legacy behaviour:
1) GetForegroundWindow, if capturable
2) else Z-order top capturable candidate
Optional prefer_pid only filters when sticky (same front app pid);
never jumps to another app just because it is larger.
3) prefer_last_active_popup (same pid)
Optional:
- under_cursor: 光标下顶层窗优先（不点击、不抢焦点）；失败则回退前台逻辑
- prefer_wecom_search_popup: 若候选为企微主窗且存在 SearchResultWindow2，改采浮层
"""
info, cands, _meta = resolve_capture_target(
exclude_hwnds=exclude_hwnds,
prefer_pid=prefer_pid,
under_cursor=under_cursor,
prefer_wecom_search_popup=prefer_wecom_search_popup,
)
return info, cands
def resolve_capture_target(
*,
exclude_hwnds: Optional[set[int]] = None,
prefer_pid: Optional[int] = None,
under_cursor: bool = False,
prefer_wecom_search_popup: bool = True,
) -> tuple[WindowInfo, list[WindowInfo], dict[str, Any]]:
"""与 find_capture_target 相同，额外返回 selection meta（strategy 链）。"""
candidates = list_capturable_candidates(exclude_hwnds=exclude_hwnds)
if not candidates:
raise RuntimeError(
@@ -548,21 +634,31 @@ def find_capture_target(
by_hwnd = {int(c.hwnd): c for c in candidates}
chosen: Optional[WindowInfo] = None
strategy_parts: list[str] = []
fg = foreground_toplevel_hwnd(exclude_hwnds=exclude_hwnds)
if fg and fg in by_hwnd:
# Frontmost always wins — never replace with a larger/sticky other app.
chosen = by_hwnd[fg]
elif fg:
# Foreground exists but filtered from candidates — still try to describe it.
try:
info = window_info_from_hwnd(fg)
if not is_inspector_window(info) and not (
is_system_chrome_window(info) and not (info.title or "").strip()
):
chosen = info
except Exception:
chosen = None
if under_cursor:
cur = window_info_under_cursor(exclude_hwnds=exclude_hwnds)
if cur is not None:
chosen = cur
strategy_parts.append("under_cursor")
if chosen is None:
fg = foreground_toplevel_hwnd(exclude_hwnds=exclude_hwnds)
if fg and fg in by_hwnd:
# Frontmost always wins — never replace with a larger/sticky other app.
chosen = by_hwnd[fg]
strategy_parts.append("foreground")
elif fg:
# Foreground exists but filtered from candidates — still try to describe it.
try:
info = window_info_from_hwnd(fg)
if not is_inspector_window(info) and not (
is_system_chrome_window(info) and not (info.title or "").strip()
):
chosen = info
strategy_parts.append("foreground_raw")
except Exception:
chosen = None
if chosen is None:
# No usable foreground: Z-order front. prefer_pid only as soft filter here.
@@ -570,15 +666,35 @@ def find_capture_target(
for c in candidates:
if int(c.pid) == int(prefer_pid):
chosen = c
strategy_parts.append("sticky_pid")
break
if chosen is None:
chosen = candidates[0]
strategy_parts.append("zorder")
# Owned popup of the *same* front window only (e.g. search panel), never another app
popup = prefer_last_active_popup(chosen)
if int(popup.pid) == int(chosen.pid):
if int(popup.pid) == int(chosen.pid) and int(popup.hwnd) != int(chosen.hwnd):
chosen = popup
return chosen, candidates
strategy_parts.append("last_active_popup")
# 企微：主窗为前台时，若搜索浮层仍打开，优先采浮层（无需点击进会话）
if prefer_wecom_search_popup and (chosen.class_name or "") == "WeWorkWindow":
search_win = find_wecom_search_result_window(
pid=int(chosen.pid), exclude_hwnds=exclude_hwnds
)
if search_win is not None:
chosen = search_win
strategy_parts.append("wecom_search_popup")
meta = {
"strategy": "+".join(strategy_parts) if strategy_parts else "unknown",
"under_cursor": bool(under_cursor),
"prefer_wecom_search_popup": bool(prefer_wecom_search_popup),
"chosen_class": chosen.class_name,
"chosen_hwnd": int(chosen.hwnd),
}
return chosen, candidates, meta
def wait_frontmost_target(
@@ -632,8 +748,6 @@ def prepare_frontmost_for_capture(
企微：先静默调用 kit ``ensure_ui_ready("wecom")``（与技能同路径），再轻量 poke。
"""
import kit_wake
info, cands, wait_meta = wait_frontmost_target(
exclude_hwnds=exclude_hwnds, timeout_sec=timeout_sec
)
@@ -642,28 +756,29 @@ def prepare_frontmost_for_capture(
# Keep operator size/position; only restore if minimized.
act = activate_main_hwnd(info.hwnd, maximize=False)
# Product adapters may need to make their accessibility provider ready
# before the operator starts typing. Running this on the app's existing
# startup worker keeps Tk responsive and prevents the first input
# transaction from racing a 10-20 second background warmup.
import kit_wake
kit_report = kit_wake.auto_ensure_if_wecom(
class_name=info.class_name or "",
title=info.title or "",
hwnd=info.hwnd,
force=True,
)
findall = kit_report.get("findall") if kit_report.get("ok") else None
provider_cold = bool(kit_report.get("applicable") and not kit_report.get("ok"))
warm = warmup_uia_provider(
info.hwnd, interact_window=False, allow_foreground_nudge=False
)
n = int(warm.get("findall_after") or findall_count_for_hwnd(info.hwnd) or 0)
if kit_report.get("ok") and int(kit_report.get("findall") or 0) > n:
n = int(kit_report["findall"])
return {
"ready": True, # frontmost resolved => ready to capture that window
"window": info.to_dict(),
"candidates": len(cands),
"findall": n,
"uia_sparse": n < 8,
"provider_cold": is_provider_cold(info.hwnd, findall=n),
"findall": findall,
"uia_sparse": bool(provider_cold),
"provider_cold": provider_cold,
"activate": act,
"warmup": warm,
"warmup": {"mode": "startup_adapter" if kit_report.get("applicable") else "deferred"},
"kit_wake": kit_report,
"wait": wait_meta,
"strategy": "foreground_only",
@@ -933,6 +1048,7 @@ def dump_window_tree(
warmup_mode:
- full: MSAA/UIA poke on target; restore only if minimized
- light: ElementFromHandle poke only (no ShowWindow)
- probe: light poke + FindAll, but never invoke a blocking product wake
- off: no provider warmup
Child / same-PID merges never call ShowWindow (light poke at most).
"""
@@ -942,7 +1058,7 @@ def dump_window_tree(
mode = (warmup_mode or "full").lower()
if not warmup:
mode = "off"
if mode not in {"full", "light", "off"}:
if mode not in {"full", "light", "probe", "off"}:
mode = "full"
kit_report: dict[str, Any] = {"applicable": False, "skipped": True, "reason": "not_checked"}
@@ -950,7 +1066,11 @@ def dump_window_tree(
winfo_for_kit = window_info_from_hwnd(hwnd)
except Exception:
winfo_for_kit = None
if winfo_for_kit is not None:
# 搜索浮层稀疏时勿合并主窗会话列表，否则验收会误判「已有数据」
allow_same_pid_merge = bool(merge_same_pid_if_sparse)
if winfo_for_kit is not None and (winfo_for_kit.class_name or "") == "SearchResultWindow2":
allow_same_pid_merge = False
if winfo_for_kit is not None and mode not in {"off", "probe"}:
hint = findall_count_for_hwnd(hwnd)
kit_report = kit_wake.auto_ensure_if_wecom(
class_name=winfo_for_kit.class_name or "",
@@ -977,7 +1097,7 @@ def dump_window_tree(
warmup_report = warmup_uia_provider(
hwnd, interact_window=True, allow_foreground_nudge=True
)
elif mode == "light":
elif mode in {"light", "probe"}:
warmup_report = poke_uia_provider(hwnd)
def _run_dump() -> tuple[dict[str, Any], list[dict[str, Any]], dict[str, Any]]:
@@ -1035,7 +1155,7 @@ def dump_window_tree(
t = dict(t)
t["children"] = children_nodes
if use_findall:
if use_findall and mode != "off":
found = collect_findall_flat(root_ctrl, max_nodes=max_nodes)
st["findall_count"] = len(found)
existing = {r.get("path_hint") for r in idx}
@@ -1047,6 +1167,13 @@ def dump_window_tree(
return t, idx, st
tree, index, stats = _run_dump()
if winfo_for_kit is not None:
kit_wake.note_provider_warm(
class_name=winfo_for_kit.class_name or "",
title=winfo_for_kit.title or "",
hwnd=hwnd,
findall=int(stats.get("findall_count") or 0),
)
stats["warmup"] = warmup_report
stats["warmup_mode"] = mode
stats["kit_wake"] = kit_report
@@ -1061,7 +1188,7 @@ def dump_window_tree(
)
# 仍冷且目标为企微：强制再 ensure 一次（绕过短时 cache）后重 dump
if stats["provider_cold"] and winfo_for_kit is not None and kit_wake.looks_like_wecom(
if mode not in {"off", "probe"} and stats["provider_cold"] and winfo_for_kit is not None and kit_wake.looks_like_wecom(
class_name=winfo_for_kit.class_name or "",
title=winfo_for_kit.title or "",
):
@@ -1117,7 +1244,9 @@ def dump_window_tree(
# Sparse fallback: light poke + FindAll on a few large same-pid top-levels.
# Includes WeWorkWindow when current target is a thin popup.
if merge_same_pid_if_sparse and stats["uia_sparse"]:
# Never merge into SearchResultWindow2 (see allow_same_pid_merge).
stats["merge_same_pid_allowed"] = allow_same_pid_merge
if allow_same_pid_merge and stats["uia_sparse"]:
try:
info = window_info_from_hwnd(hwnd)
merged_n = 0
@@ -1329,15 +1458,20 @@ def format_tree_text(tree: dict[str, Any], *, max_lines: int = 2000) -> str:
return "\n".join(lines)
# Back-compat alias used by older app versions
def resolve_capture_target(
# Back-compat: older callers expected a single WindowInfo (foreground only).
# Do NOT name this resolve_capture_target — that name is the full 3-tuple API above.
def resolve_frontmost_window(
*,
self_hwnd: int = 0,
poll_sec: float = 0.15,
wait_sec: float = 2.0,
) -> WindowInfo:
_ = poll_sec, wait_sec
info, _cands = find_capture_target(exclude_hwnds={int(self_hwnd)} if self_hwnd else None)
info, _cands = find_capture_target(
exclude_hwnds={int(self_hwnd)} if self_hwnd else None,
under_cursor=False,
prefer_wecom_search_popup=False,
)
return info