chore: archive tools/ui_inspector snapshot before migration removal.
Keep the last in-template copy on main so history can restore it; product continues in jiangchang-ui-recorder. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
6
tools/ui_inspector/diagnostics/__init__.py
Normal file
6
tools/ui_inspector/diagnostics/__init__.py
Normal file
@@ -0,0 +1,6 @@
|
||||
"""Capture validation and timeline diagnostics."""
|
||||
|
||||
from .capture_validator import validate_capture
|
||||
from .replay_preflight import build_replay_preflight, write_replay_preflight
|
||||
|
||||
__all__ = ["validate_capture", "build_replay_preflight", "write_replay_preflight"]
|
||||
134
tools/ui_inspector/diagnostics/capture_validator.py
Normal file
134
tools/ui_inspector/diagnostics/capture_validator.py
Normal file
@@ -0,0 +1,134 @@
|
||||
"""Product-neutral capture quality checks."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _read(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def validate_capture(flow_dir: Path) -> dict[str, Any]:
|
||||
root = Path(flow_dir)
|
||||
issues: list[dict[str, Any]] = []
|
||||
actions = sorted(
|
||||
(_read(path) for path in (root / "actions").glob("*.json")),
|
||||
key=lambda item: int(item.get("sequence") or 0),
|
||||
)
|
||||
expected = list(range(1, len(actions) + 1))
|
||||
actual = [int(item.get("sequence") or 0) for item in actions]
|
||||
if actual != expected:
|
||||
issues.append({"severity": "error", "code": "ACTION_SEQUENCE_GAP", "actual": actual})
|
||||
elements = {
|
||||
item.get("id"): item
|
||||
for item in (_read(path) for path in (root / "elements").glob("*.json"))
|
||||
if item.get("id")
|
||||
}
|
||||
interactive_types = {"click", "fill", "type", "press_key", "select", "toggle"}
|
||||
previous_action: dict[str, Any] | None = None
|
||||
for action in actions:
|
||||
target = dict(action.get("target_snapshot") or {})
|
||||
rect = dict(target.get("rect") or target.get("bounds") or {})
|
||||
area = float(rect.get("width") or 0) * float(rect.get("height") or 0)
|
||||
if action.get("action_type") in {"click", "fill", "select", "toggle"} and not target:
|
||||
issues.append({"severity": "error", "code": "ACTION_TARGET_MISSING", "action_id": action.get("id")})
|
||||
if action.get("action_type") == "click" and area > 1_000_000:
|
||||
issues.append({"severity": "error", "code": "CLICK_TARGET_OVERSIZED", "action_id": action.get("id"), "area": area})
|
||||
target_name = str(target.get("name") or target.get("automation_id") or "").lower()
|
||||
if (
|
||||
action.get("surface") == "windows"
|
||||
and action.get("action_type") == "click"
|
||||
and str(target.get("control_type") or "") in {"TreeControl", "PaneControl", "GroupControl"}
|
||||
and any(token in target_name for token in ("search_result", "listview", "list_view", "watermark", "container", "layout"))
|
||||
):
|
||||
issues.append({"severity": "warning", "code": "CLICK_TARGET_GENERIC_CONTAINER", "action_id": action.get("id"), "target_name": target_name})
|
||||
if (
|
||||
action.get("surface") == "windows"
|
||||
and action.get("action_type") == "press_key"
|
||||
and (str(target.get("control_type") or "") == "WindowControl" or area > 1_000_000)
|
||||
):
|
||||
issues.append({"severity": "warning", "code": "KEY_TARGET_ROOT_WINDOW", "action_id": action.get("id")})
|
||||
if action.get("action_type") == "click" and str(target.get("tag") or "").lower() in {"path", "use", "g", "svg"}:
|
||||
issues.append({"severity": "warning", "code": "GRAPHIC_LEAF_TARGET", "action_id": action.get("id"), "tag": target.get("tag")})
|
||||
evidence = dict(action.get("evidence") or {})
|
||||
if not action.get("after_observation_id") and not evidence.get("eventual_observation_id"):
|
||||
issues.append({"severity": "warning", "code": "ACTION_EVIDENCE_MISSING", "action_id": action.get("id")})
|
||||
if evidence.get("locator_unique") is False:
|
||||
issues.append({"severity": "warning", "code": "LOCATOR_NOT_UNIQUE", "action_id": action.get("id")})
|
||||
if action.get("surface") == "browser" and action.get("action_type") in interactive_types:
|
||||
element_id = action.get("target_element_id")
|
||||
if target and not element_id:
|
||||
issues.append({"severity": "warning", "code": "TARGET_ELEMENT_REFERENCE_MISSING", "action_id": action.get("id")})
|
||||
element = elements.get(element_id) if element_id else None
|
||||
primary = dict((element or {}).get("primary_locator") or {})
|
||||
if element_id and (not primary or not primary.get("production_eligible")):
|
||||
issues.append({"severity": "warning", "code": "ACTION_PRODUCTION_LOCATOR_MISSING", "action_id": action.get("id"), "element_id": element_id})
|
||||
elif primary and float(primary.get("stability_score") or 0.0) < 0.55:
|
||||
issues.append({"severity": "warning", "code": "LOW_STABILITY_LOCATOR", "action_id": action.get("id"), "element_id": element_id, "strategy": primary.get("strategy"), "stability_score": primary.get("stability_score")})
|
||||
raw_meta = dict((action.get("raw_event") or {}).get("meta") or {})
|
||||
if not raw_meta.get("event_id"):
|
||||
issues.append({"severity": "warning", "code": "ACTION_RAW_EVENT_LINK_MISSING", "action_id": action.get("id")})
|
||||
if action.get("surface") == "windows" and action.get("action_type") in interactive_types:
|
||||
element_id = action.get("target_element_id")
|
||||
if target and not element_id:
|
||||
issues.append({"severity": "warning", "code": "TARGET_ELEMENT_REFERENCE_MISSING", "action_id": action.get("id")})
|
||||
element = elements.get(element_id) if element_id else None
|
||||
primary = dict((element or {}).get("primary_locator") or {})
|
||||
if element_id and not primary:
|
||||
issues.append({"severity": "warning", "code": "ACTION_PRODUCTION_LOCATOR_MISSING", "action_id": action.get("id"), "element_id": element_id})
|
||||
if previous_action and action.get("action_type") == "fill" and previous_action.get("action_type") == "fill":
|
||||
same_target = bool(action.get("target_element_id")) and action.get("target_element_id") == previous_action.get("target_element_id")
|
||||
old_text = str((previous_action.get("input") or {}).get("text") or "")
|
||||
new_text = str((action.get("input") or {}).get("text") or "")
|
||||
old_ms = ((previous_action.get("raw_event") or {}).get("meta") or {}).get("source_ts_ms")
|
||||
new_ms = ((action.get("raw_event") or {}).get("meta") or {}).get("source_ts_ms")
|
||||
close = False
|
||||
try:
|
||||
close = 0 <= float(new_ms) - float(old_ms) <= 2000
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
if same_target and old_text and new_text.startswith(old_text) and new_text != old_text and close:
|
||||
issues.append({"severity": "warning", "code": "DUPLICATE_FILL_TRANSACTION", "action_id": action.get("id"), "previous_action_id": previous_action.get("id"), "element_id": action.get("target_element_id")})
|
||||
previous_action = action
|
||||
journal = root / "journal" / "raw_events.jsonl"
|
||||
if not journal.exists():
|
||||
issues.append({"severity": "error", "code": "RAW_EVENT_JOURNAL_MISSING"})
|
||||
if not actions:
|
||||
issues.append({"severity": "error", "code": "ZERO_ACTIONS"})
|
||||
typing_transactions = 0
|
||||
if journal.exists():
|
||||
for line in journal.read_text(encoding="utf-8").splitlines():
|
||||
try:
|
||||
payload = json.loads(line)
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
continue
|
||||
event = dict(payload.get("event") or payload)
|
||||
if event.get("kind") == "typing_activity":
|
||||
typing_transactions += 1
|
||||
fill_count = sum(
|
||||
1 for action in actions if action.get("action_type") in {"fill", "type"}
|
||||
)
|
||||
if typing_transactions > fill_count:
|
||||
issues.append(
|
||||
{
|
||||
"severity": "warning",
|
||||
"code": "TEXT_INPUT_TRANSACTION_MISSING",
|
||||
"typing_transactions": typing_transactions,
|
||||
"fill_actions": fill_count,
|
||||
}
|
||||
)
|
||||
screenshot_count = len(list((root / "observations").glob("*/screen.png")))
|
||||
return {
|
||||
"schema": "openclaw.ui_inspector.validation.v1",
|
||||
"flow_dir": str(root),
|
||||
"action_count": len(actions),
|
||||
"element_count": len(elements),
|
||||
"observation_count": len(list((root / "observations").glob("*/meta.json"))),
|
||||
"screenshot_count": screenshot_count,
|
||||
"error_count": sum(1 for issue in issues if issue["severity"] == "error"),
|
||||
"warning_count": sum(1 for issue in issues if issue["severity"] == "warning"),
|
||||
"issues": issues,
|
||||
}
|
||||
133
tools/ui_inspector/diagnostics/replay_preflight.py
Normal file
133
tools/ui_inspector/diagnostics/replay_preflight.py
Normal file
@@ -0,0 +1,133 @@
|
||||
"""Static, non-mutating replay readiness report for a finished capture."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
_INTERACTIVE = {"click", "fill", "type", "press_key", "select", "toggle"}
|
||||
|
||||
|
||||
def _read(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def build_replay_preflight(flow_dir: Path) -> dict[str, Any]:
|
||||
"""Assess whether recorded actions contain executable, verified locators.
|
||||
|
||||
This deliberately does not click or navigate. `event_time_unique` means
|
||||
the locator count was measured in the event-origin document during capture;
|
||||
it is not represented as a cross-session replay result.
|
||||
"""
|
||||
root = Path(flow_dir)
|
||||
actions = sorted(
|
||||
(_read(path) for path in (root / "actions").glob("*.json")),
|
||||
key=lambda item: int(item.get("sequence") or 0),
|
||||
)
|
||||
elements = {
|
||||
item.get("id"): item
|
||||
for item in (_read(path) for path in (root / "elements").glob("*.json"))
|
||||
if item.get("id")
|
||||
}
|
||||
rows: list[dict[str, Any]] = []
|
||||
for action in actions:
|
||||
action_type = str(action.get("action_type") or "")
|
||||
if action_type not in _INTERACTIVE:
|
||||
rows.append({
|
||||
"action_id": action.get("id"),
|
||||
"sequence": action.get("sequence"),
|
||||
"action_type": action_type,
|
||||
"result": "context_only",
|
||||
"reason": "non_interactive_or_navigation",
|
||||
})
|
||||
continue
|
||||
element_id = action.get("target_element_id")
|
||||
element = elements.get(element_id) or {}
|
||||
primary = dict(element.get("primary_locator") or {})
|
||||
eligible = bool(primary.get("production_eligible"))
|
||||
unique = primary.get("unique") is True
|
||||
score = float(primary.get("stability_score") or 0.0)
|
||||
result = "pass" if eligible and unique and score >= 0.55 else "warning"
|
||||
if not eligible or not unique:
|
||||
result = "fail"
|
||||
target = dict(action.get("target_snapshot") or {})
|
||||
bounds = dict(target.get("bounds") or target.get("rect") or {})
|
||||
area = float(bounds.get("width") or 0) * float(bounds.get("height") or 0)
|
||||
oversized_click = action_type == "click" and area > 1_000_000
|
||||
root_window_target = (
|
||||
action_type == "click"
|
||||
and str(target.get("control_type") or "") == "WindowControl"
|
||||
)
|
||||
if oversized_click or root_window_target:
|
||||
result = "fail"
|
||||
validation = str(primary.get("validation") or "none")
|
||||
verification = (
|
||||
"event_time_unique" if validation == "live" and unique
|
||||
else "snapshot_unique" if validation in {"snapshot_only", "live_tree"} and unique
|
||||
else "not_validated"
|
||||
)
|
||||
raw_meta = dict((action.get("raw_event") or {}).get("meta") or {})
|
||||
rows.append({
|
||||
"action_id": action.get("id"),
|
||||
"sequence": action.get("sequence"),
|
||||
"action_type": action_type,
|
||||
"result": result,
|
||||
"element_id": element_id,
|
||||
"logical_name": element.get("logical_name") or "",
|
||||
"locator": primary.get("expression") or "",
|
||||
"strategy": primary.get("strategy") or "",
|
||||
"stability_score": score,
|
||||
"match_count": primary.get("actual_count"),
|
||||
"verification": verification,
|
||||
"page_id": raw_meta.get("page_id") or "",
|
||||
"document_id": raw_meta.get("document_id") or "",
|
||||
"raw_event_ids": list(action.get("raw_event_ids") or []),
|
||||
"reason": (
|
||||
"oversized_or_root_window_click_target"
|
||||
if oversized_click or root_window_target else ""
|
||||
),
|
||||
})
|
||||
counts = {
|
||||
name: sum(1 for row in rows if row.get("result") == name)
|
||||
for name in ("pass", "warning", "fail", "context_only")
|
||||
}
|
||||
from diagnostics.capture_validator import validate_capture
|
||||
|
||||
quality = validate_capture(root)
|
||||
blocking_codes = {
|
||||
"TEXT_INPUT_TRANSACTION_MISSING",
|
||||
"CLICK_TARGET_GENERIC_CONTAINER",
|
||||
"KEY_TARGET_ROOT_WINDOW",
|
||||
"CLICK_TARGET_OVERSIZED",
|
||||
"ACTION_TARGET_MISSING",
|
||||
"ZERO_ACTIONS",
|
||||
}
|
||||
quality_blockers = [
|
||||
issue
|
||||
for issue in quality.get("issues") or []
|
||||
if issue.get("severity") == "error" or issue.get("code") in blocking_codes
|
||||
]
|
||||
counts["quality_blockers"] = len(quality_blockers)
|
||||
return {
|
||||
"schema": "openclaw.ui_inspector.replay_preflight.v1",
|
||||
"mode": "static_non_mutating",
|
||||
"note": "事件时刻唯一性不等同于跨会话真实回放；本报告不会点击、输入或导航。",
|
||||
"flow_dir": str(root),
|
||||
"ready": not quality_blockers and counts.get("fail", 0) == 0,
|
||||
"summary": counts,
|
||||
"quality_gate": {
|
||||
"passed": not quality_blockers,
|
||||
"issues": quality_blockers,
|
||||
},
|
||||
"actions": rows,
|
||||
}
|
||||
|
||||
|
||||
def write_replay_preflight(flow_dir: Path) -> dict[str, Any]:
|
||||
report = build_replay_preflight(flow_dir)
|
||||
path = Path(flow_dir) / "diagnostics" / "replay_report.json"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
return report
|
||||
Reference in New Issue
Block a user