chendelian
633ecff4e2
Keep the last in-template copy on main so history can restore it; product continues in jiangchang-ui-recorder. Co-authored-by: Cursor <cursoragent@cursor.com>
135 lines
7.9 KiB
Python
135 lines
7.9 KiB
Python
"""Product-neutral capture quality checks."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
def _read(path: Path) -> dict[str, Any]:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def validate_capture(flow_dir: Path) -> dict[str, Any]:
|
|
root = Path(flow_dir)
|
|
issues: list[dict[str, Any]] = []
|
|
actions = sorted(
|
|
(_read(path) for path in (root / "actions").glob("*.json")),
|
|
key=lambda item: int(item.get("sequence") or 0),
|
|
)
|
|
expected = list(range(1, len(actions) + 1))
|
|
actual = [int(item.get("sequence") or 0) for item in actions]
|
|
if actual != expected:
|
|
issues.append({"severity": "error", "code": "ACTION_SEQUENCE_GAP", "actual": actual})
|
|
elements = {
|
|
item.get("id"): item
|
|
for item in (_read(path) for path in (root / "elements").glob("*.json"))
|
|
if item.get("id")
|
|
}
|
|
interactive_types = {"click", "fill", "type", "press_key", "select", "toggle"}
|
|
previous_action: dict[str, Any] | None = None
|
|
for action in actions:
|
|
target = dict(action.get("target_snapshot") or {})
|
|
rect = dict(target.get("rect") or target.get("bounds") or {})
|
|
area = float(rect.get("width") or 0) * float(rect.get("height") or 0)
|
|
if action.get("action_type") in {"click", "fill", "select", "toggle"} and not target:
|
|
issues.append({"severity": "error", "code": "ACTION_TARGET_MISSING", "action_id": action.get("id")})
|
|
if action.get("action_type") == "click" and area > 1_000_000:
|
|
issues.append({"severity": "error", "code": "CLICK_TARGET_OVERSIZED", "action_id": action.get("id"), "area": area})
|
|
target_name = str(target.get("name") or target.get("automation_id") or "").lower()
|
|
if (
|
|
action.get("surface") == "windows"
|
|
and action.get("action_type") == "click"
|
|
and str(target.get("control_type") or "") in {"TreeControl", "PaneControl", "GroupControl"}
|
|
and any(token in target_name for token in ("search_result", "listview", "list_view", "watermark", "container", "layout"))
|
|
):
|
|
issues.append({"severity": "warning", "code": "CLICK_TARGET_GENERIC_CONTAINER", "action_id": action.get("id"), "target_name": target_name})
|
|
if (
|
|
action.get("surface") == "windows"
|
|
and action.get("action_type") == "press_key"
|
|
and (str(target.get("control_type") or "") == "WindowControl" or area > 1_000_000)
|
|
):
|
|
issues.append({"severity": "warning", "code": "KEY_TARGET_ROOT_WINDOW", "action_id": action.get("id")})
|
|
if action.get("action_type") == "click" and str(target.get("tag") or "").lower() in {"path", "use", "g", "svg"}:
|
|
issues.append({"severity": "warning", "code": "GRAPHIC_LEAF_TARGET", "action_id": action.get("id"), "tag": target.get("tag")})
|
|
evidence = dict(action.get("evidence") or {})
|
|
if not action.get("after_observation_id") and not evidence.get("eventual_observation_id"):
|
|
issues.append({"severity": "warning", "code": "ACTION_EVIDENCE_MISSING", "action_id": action.get("id")})
|
|
if evidence.get("locator_unique") is False:
|
|
issues.append({"severity": "warning", "code": "LOCATOR_NOT_UNIQUE", "action_id": action.get("id")})
|
|
if action.get("surface") == "browser" and action.get("action_type") in interactive_types:
|
|
element_id = action.get("target_element_id")
|
|
if target and not element_id:
|
|
issues.append({"severity": "warning", "code": "TARGET_ELEMENT_REFERENCE_MISSING", "action_id": action.get("id")})
|
|
element = elements.get(element_id) if element_id else None
|
|
primary = dict((element or {}).get("primary_locator") or {})
|
|
if element_id and (not primary or not primary.get("production_eligible")):
|
|
issues.append({"severity": "warning", "code": "ACTION_PRODUCTION_LOCATOR_MISSING", "action_id": action.get("id"), "element_id": element_id})
|
|
elif primary and float(primary.get("stability_score") or 0.0) < 0.55:
|
|
issues.append({"severity": "warning", "code": "LOW_STABILITY_LOCATOR", "action_id": action.get("id"), "element_id": element_id, "strategy": primary.get("strategy"), "stability_score": primary.get("stability_score")})
|
|
raw_meta = dict((action.get("raw_event") or {}).get("meta") or {})
|
|
if not raw_meta.get("event_id"):
|
|
issues.append({"severity": "warning", "code": "ACTION_RAW_EVENT_LINK_MISSING", "action_id": action.get("id")})
|
|
if action.get("surface") == "windows" and action.get("action_type") in interactive_types:
|
|
element_id = action.get("target_element_id")
|
|
if target and not element_id:
|
|
issues.append({"severity": "warning", "code": "TARGET_ELEMENT_REFERENCE_MISSING", "action_id": action.get("id")})
|
|
element = elements.get(element_id) if element_id else None
|
|
primary = dict((element or {}).get("primary_locator") or {})
|
|
if element_id and not primary:
|
|
issues.append({"severity": "warning", "code": "ACTION_PRODUCTION_LOCATOR_MISSING", "action_id": action.get("id"), "element_id": element_id})
|
|
if previous_action and action.get("action_type") == "fill" and previous_action.get("action_type") == "fill":
|
|
same_target = bool(action.get("target_element_id")) and action.get("target_element_id") == previous_action.get("target_element_id")
|
|
old_text = str((previous_action.get("input") or {}).get("text") or "")
|
|
new_text = str((action.get("input") or {}).get("text") or "")
|
|
old_ms = ((previous_action.get("raw_event") or {}).get("meta") or {}).get("source_ts_ms")
|
|
new_ms = ((action.get("raw_event") or {}).get("meta") or {}).get("source_ts_ms")
|
|
close = False
|
|
try:
|
|
close = 0 <= float(new_ms) - float(old_ms) <= 2000
|
|
except (TypeError, ValueError):
|
|
pass
|
|
if same_target and old_text and new_text.startswith(old_text) and new_text != old_text and close:
|
|
issues.append({"severity": "warning", "code": "DUPLICATE_FILL_TRANSACTION", "action_id": action.get("id"), "previous_action_id": previous_action.get("id"), "element_id": action.get("target_element_id")})
|
|
previous_action = action
|
|
journal = root / "journal" / "raw_events.jsonl"
|
|
if not journal.exists():
|
|
issues.append({"severity": "error", "code": "RAW_EVENT_JOURNAL_MISSING"})
|
|
if not actions:
|
|
issues.append({"severity": "error", "code": "ZERO_ACTIONS"})
|
|
typing_transactions = 0
|
|
if journal.exists():
|
|
for line in journal.read_text(encoding="utf-8").splitlines():
|
|
try:
|
|
payload = json.loads(line)
|
|
except (TypeError, ValueError, json.JSONDecodeError):
|
|
continue
|
|
event = dict(payload.get("event") or payload)
|
|
if event.get("kind") == "typing_activity":
|
|
typing_transactions += 1
|
|
fill_count = sum(
|
|
1 for action in actions if action.get("action_type") in {"fill", "type"}
|
|
)
|
|
if typing_transactions > fill_count:
|
|
issues.append(
|
|
{
|
|
"severity": "warning",
|
|
"code": "TEXT_INPUT_TRANSACTION_MISSING",
|
|
"typing_transactions": typing_transactions,
|
|
"fill_actions": fill_count,
|
|
}
|
|
)
|
|
screenshot_count = len(list((root / "observations").glob("*/screen.png")))
|
|
return {
|
|
"schema": "openclaw.ui_inspector.validation.v1",
|
|
"flow_dir": str(root),
|
|
"action_count": len(actions),
|
|
"element_count": len(elements),
|
|
"observation_count": len(list((root / "observations").glob("*/meta.json"))),
|
|
"screenshot_count": screenshot_count,
|
|
"error_count": sum(1 for issue in issues if issue["severity"] == "error"),
|
|
"warning_count": sum(1 for issue in issues if issue["severity"] == "warning"),
|
|
"issues": issues,
|
|
}
|