diff --git a/references/RPA.md b/references/RPA.md index b4f532f..964f9bb 100644 --- a/references/RPA.md +++ b/references/RPA.md @@ -159,10 +159,11 @@ skill 退出/抛错统一用 `ERROR:` 前缀 + 稳定码，方便宿主与上层 - **路径**：`{CLAW_DATA_ROOT}/{user}/{slug}/rpa-artifacts/{batch_id}/{tag}_{timestamp}.png` - **失败必截图**：受 `OPENCLAW_ARTIFACTS_ON_FAILURE`（默认开）控制。 -- **运行录屏成片**：RPA 型技能默认 `OPENCLAW_RECORD_VIDEO=1`，表示录屏、中文字幕、背景音乐、最终 MP4 **一体开启**。由 service 核心中的 `RpaVideoSession` 统一触发（见 `jiangchang_skill_core.rpa.video_session`），不在 CLI/E2E 入口单独实现。 -- **最终视频**：`{skill_data_dir}/videos/{skill_slug}_{yyyyMMdd_HHmmss}_{batch_id}.mp4`。 -- **中间产物**：`{skill_data_dir}/rpa-artifacts/{batch_id}/`（raw-video、字幕、失败截图等），**不放最终 MP4**。 -- Playwright 使用 `record_video_dir` 指向 `rpa-artifacts/{batch_id}/raw-video/`。 +- **运行录屏成片**：RPA 型技能默认 `OPENCLAW_RECORD_VIDEO=1`，表示 **ffmpeg 录制运行过程** + 字幕 + 背景音乐 + 最终 MP4 **一体开启**。由 `RpaVideoSession` 统一触发，不在 CLI/E2E 单独实现。 +- **Playwright 不负责录屏**，仅浏览器自动化。 +- **ffmpeg 是唯一录屏器**（Windows：`gdigrab` + `desktop`）。 +- **最终视频**：`{skill_data_dir}/videos/{skill_slug}_{yyyyMMdd_HHmmss}_{batch_id}.mp4` +- **中间产物**：`rpa-artifacts/{batch_id}/capture.mp4`、`subtitles/`、`logs/` 等，**不放最终 MP4**。 - **常见 tag**：`before_submit` / `after_submit` / `captcha` / `login_fail` / `error`。 --- diff --git a/scripts/jiangchang_skill_core/rpa/browser.py b/scripts/jiangchang_skill_core/rpa/browser.py index 4013a22..8d4ec77 100644 --- a/scripts/jiangchang_skill_core/rpa/browser.py +++ b/scripts/jiangchang_skill_core/rpa/browser.py @@ -1,5 +1,4 @@ -# vendored from jiangchang-platform-kit; do not edit here, edit the kit. -"""统一 persistent context 启动封装。""" +"""统一 persistent context 启动封装（仅浏览器自动化，不负责录屏）。""" from __future__ import annotations @@ -25,12 +24,11 @@ async def launch_persistent_browser( executable_path: str | None = None, headless: bool | None = None, extra_args: list[str] | None = None, - record_video_dir: str | None = None, ) -> "BrowserContext": """用 stealth 参数启动 persistent context + 注入 STEALTH_INIT_SCRIPT。 headless=None 时读 OPENCLAW_BROWSER_HEADLESS。 - record_video_dir 非空则开录屏。 + 录屏由 RpaVideoSession（ffmpeg）负责，本函数不向 Playwright 传递任何录屏参数。 """ if headless is None: v = (os.environ.get("OPENCLAW_BROWSER_HEADLESS") or "0").strip().lower() @@ -53,8 +51,6 @@ async def launch_persistent_browser( ) if ignore is not None: launch_kwargs["ignore_default_args"] = ignore - if record_video_dir: - launch_kwargs["record_video_dir"] = record_video_dir context = await playwright.chromium.launch_persistent_context(**launch_kwargs) if stealth_enabled(): diff --git a/scripts/jiangchang_skill_core/rpa/video_session.py b/scripts/jiangchang_skill_core/rpa/video_session.py index bcc4ea2..206d4d7 100644 --- a/scripts/jiangchang_skill_core/rpa/video_session.py +++ b/scripts/jiangchang_skill_core/rpa/video_session.py @@ -1,10 +1,14 @@ -"""RPA 运行录屏会话：Playwright raw video → 字幕 + 背景音乐 → 最终 MP4。""" +"""RPA 运行录屏会话：ffmpeg 桌面录制 → 字幕 + 背景音乐 → 最终 MP4。 + +Playwright 仅用于浏览器自动化，不负责录屏。 +""" from __future__ import annotations import asyncio import os import random +import shutil import subprocess import sys import time @@ -15,7 +19,6 @@ from typing import Any, Dict, List, Optional from .. import config -# 与 screencast/composer.py 保持一致的字幕样式 _JIANGCHANG_SUBTITLE_STYLE = ( "FontName=Arial,FontSize=14," "PrimaryColour=&H00FFFFFF,OutlineColour=&H00000000," @@ -27,6 +30,8 @@ _DEFAULT_MUSIC_ROOT = os.environ.get( r"D:\OpenClaw\client-commons\media-assets", ) +_CAPTURE_FRAMERATE = 15 + def _record_video_enabled() -> bool: return config.get_bool("OPENCLAW_RECORD_VIDEO", default=False) @@ -50,7 +55,6 @@ class _StepEntry: def _generate_srt(entries: List[_StepEntry], output_path: str) -> None: - """根据步骤时间线生成 SRT（防重叠）。""" MIN_DURATION = 1.0 GAP = 0.1 steps = sorted(entries, key=lambda e: e.start) @@ -86,16 +90,6 @@ def _escape_srt_path(path: str) -> str: return path.replace("\\", "/").replace(":", "\\:") -def _find_raw_webm(raw_dir: str) -> Optional[Path]: - root = Path(raw_dir) - if not root.is_dir(): - return None - webms = list(root.rglob("*.webm")) - if not webms: - return None - return max(webms, key=lambda p: p.stat().st_size) - - def _run_ffmpeg(cmd: List[str]) -> tuple[bool, str]: try: result = subprocess.run( @@ -113,8 +107,31 @@ def _run_ffmpeg(cmd: List[str]) -> tuple[bool, str]: return True, "" -def _compose_webm_mp4( - webm_path: Path, +def _build_desktop_record_cmd(capture_path: str) -> Optional[List[str]]: + """Windows gdigrab 全桌面录制；非 Windows 返回 None。""" + if sys.platform != "win32": + return None + return [ + "ffmpeg", + "-y", + "-f", + "gdigrab", + "-framerate", + str(_CAPTURE_FRAMERATE), + "-i", + "desktop", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + capture_path, + ] + + +def _compose_capture_mp4( + capture_path: Path, srt_path: Path, output_path: Path, *, @@ -134,39 +151,60 @@ def _compose_webm_mp4( f"[1:a]volume={music_volume},apad[aout]" ) cmd = [ - "ffmpeg", "-y", - "-i", str(webm_path), - "-i", str(music_file), - "-filter_complex", filter_complex, - "-map", "[vout]", - "-map", "[aout]", - "-c:v", "libx264", - "-preset", "fast", - "-crf", "23", - "-c:a", "aac", - "-b:a", "128k", + "ffmpeg", + "-y", + "-i", + str(capture_path), + "-i", + str(music_file), + "-filter_complex", + filter_complex, + "-map", + "[vout]", + "-map", + "[aout]", + "-c:v", + "libx264", + "-preset", + "fast", + "-crf", + "23", + "-c:a", + "aac", + "-b:a", + "128k", "-shortest", - "-pix_fmt", "yuv420p", + "-pix_fmt", + "yuv420p", str(output_path), ] else: cmd = [ - "ffmpeg", "-y", - "-i", str(webm_path), - "-vf", subtitle_filter, - "-c:v", "libx264", - "-preset", "fast", - "-crf", "23", - "-c:a", "aac", - "-b:a", "128k", - "-pix_fmt", "yuv420p", + "ffmpeg", + "-y", + "-i", + str(capture_path), + "-vf", + subtitle_filter, + "-c:v", + "libx264", + "-preset", + "fast", + "-crf", + "23", + "-c:a", + "aac", + "-b:a", + "128k", + "-pix_fmt", + "yuv420p", str(output_path), ] return _run_ffmpeg(cmd) class RpaVideoSession: - """Service/RPA 层统一录屏会话；OPENCLAW_RECORD_VIDEO=1 时启用完整成片流程。""" + """Service/RPA 层统一 ffmpeg 桌面录屏；OPENCLAW_RECORD_VIDEO=1 时启用成片流程。""" def __init__( self, @@ -183,6 +221,8 @@ class RpaVideoSession: self.enabled = _record_video_enabled() self.warnings: List[str] = [] self.output_video_path: Optional[str] = None + self.capture_path: Optional[str] = None + # 已废弃：勿用于 Playwright；始终为 None。 self.record_video_dir: Optional[str] = None self.artifacts: Dict[str, Any] = {} @@ -194,24 +234,27 @@ class RpaVideoSession: self._artifact_root = os.path.join( self.skill_data_dir, "rpa-artifacts", batch_id ) - self._raw_video_dir = os.path.join(self._artifact_root, "raw-video") self._subtitles_dir = os.path.join(self._artifact_root, "subtitles") self._logs_dir = os.path.join(self._artifact_root, "logs") + self._capture_path = os.path.join(self._artifact_root, "capture.mp4") self._srt_path = os.path.join(self._subtitles_dir, f"{self._video_stem}.srt") self._planned_output = os.path.join(self._videos_dir, f"{self._video_stem}.mp4") self._steps: List[_StepEntry] = [] self._t0: Optional[float] = None + self._ffmpeg_proc: Optional[subprocess.Popen] = None if self.enabled: os.makedirs(self._videos_dir, exist_ok=True) - os.makedirs(self._raw_video_dir, exist_ok=True) os.makedirs(self._subtitles_dir, exist_ok=True) os.makedirs(self._logs_dir, exist_ok=True) - self.record_video_dir = self._raw_video_dir + os.makedirs(self._artifact_root, exist_ok=True) def step(self, text: str, *, duration: float = 4.0) -> None: - """记录一步骤字幕（中文说明）。""" + self.add_step(text, duration=duration) + + def add_step(self, text: str, *, duration: float = 4.0) -> None: + """记录一步骤字幕（相对录屏开始时间）。""" if not self.enabled: return if self._t0 is None: @@ -219,71 +262,159 @@ class RpaVideoSession: elapsed = time.monotonic() - self._t0 self._steps.append(_StepEntry(start=elapsed, text=text.strip(), duration=duration)) + def _start_ffmpeg_capture(self) -> None: + if not self.enabled: + return + if not shutil.which("ffmpeg"): + self.warnings.append("ffmpeg_not_found") + return + + cmd = _build_desktop_record_cmd(self._capture_path) + if cmd is None: + self.warnings.append( + "ffmpeg_record_start_failed: desktop capture only supported on Windows" + ) + return + + try: + self._ffmpeg_proc = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + self.capture_path = self._capture_path + except Exception as exc: + self.warnings.append(f"ffmpeg_record_start_failed: {exc}") + self._ffmpeg_proc = None + + def _stop_ffmpeg_capture(self) -> None: + proc = self._ffmpeg_proc + if proc is None: + return + if proc.poll() is None: + try: + if proc.stdin: + proc.stdin.write(b"q") + proc.stdin.flush() + except Exception: + try: + proc.terminate() + except Exception: + pass + try: + proc.wait(timeout=20) + except subprocess.TimeoutExpired: + try: + proc.kill() + except Exception: + pass + try: + proc.wait(timeout=5) + except Exception: + pass + self._ffmpeg_proc = None + await_sleep = 0.3 + time.sleep(await_sleep) + async def __aenter__(self) -> "RpaVideoSession": if self.enabled: self._t0 = time.monotonic() if self.title: - self.step(self.title, duration=3.0) + self.add_step(self.title, duration=3.0) + self._start_ffmpeg_capture() return self async def __aexit__(self, exc_type, exc, tb) -> None: if self.enabled: + self._stop_ffmpeg_capture() await self.finalize() async def finalize(self) -> None: - """在 Playwright context 已 close 后合成最终 MP4；失败只记 warning。""" + """停止录制后合成最终 MP4；失败只记 warning，不抛异常。""" if not self.enabled: + self.artifacts = self.summary() return if not self._steps and self.title: - self.step(self.title) + self.add_step(self.title) try: - await asyncio.sleep(0.5) - webm = _find_raw_webm(self._raw_video_dir) - if webm is None: - self.warnings.append("no_playwright_raw_video") - self.artifacts = self.to_artifact_dict() + capture = Path(self._capture_path) + if not capture.is_file() or capture.stat().st_size <= 0: + self.warnings.append("ffmpeg_capture_missing") + self.artifacts = self.summary() return + self.capture_path = str(capture.resolve()) + entries = list(self._steps) if entries: entries[-1].duration = max(entries[-1].duration, 5.0) - _generate_srt(entries, self._srt_path) + try: + _generate_srt(entries, self._srt_path) + except Exception as exc: + self.warnings.append(f"subtitle_write_failed: {exc}") music = _resolve_music_file() if music is None: self.warnings.append("background_music_missing") - ok, err = _compose_webm_mp4( - webm, - Path(self._srt_path), + srt_file = Path(self._srt_path) + use_srt = srt_file.is_file() and srt_file.stat().st_size > 0 + + ok, err = _compose_capture_mp4( + capture, + srt_file if use_srt else Path(self._srt_path), Path(self._planned_output), music_file=music, ) if not ok and music is not None: self.warnings.append(f"ffmpeg_with_music_failed: {err}") - ok, err = _compose_webm_mp4( - webm, - Path(self._srt_path), + ok, err = _compose_capture_mp4( + capture, + srt_file if use_srt else Path(self._srt_path), Path(self._planned_output), music_file=None, ) + if not ok and not use_srt: + ok, err = _run_ffmpeg( + [ + "ffmpeg", + "-y", + "-i", + str(capture), + "-c:v", + "libx264", + "-preset", + "fast", + "-crf", + "23", + "-pix_fmt", + "yuv420p", + str(self._planned_output), + ] + ) if not ok: self.warnings.append(f"ffmpeg_compose_failed: {err}") - self.artifacts = self.to_artifact_dict() + self.artifacts = self.summary() return self.output_video_path = str(Path(self._planned_output).resolve()) - self.artifacts = self.to_artifact_dict() except Exception as exc: self.warnings.append(f"video_finalize_error: {exc}") - self.artifacts = self.to_artifact_dict() - def to_artifact_dict(self) -> Dict[str, Any]: + self.artifacts = self.summary() + + def summary(self) -> Dict[str, Any]: return { "enabled": self.enabled, "path": self.output_video_path, + "capture_path": self.capture_path, "warnings": list(self.warnings), - "batch_id": self.batch_id, } + + def to_artifact_dict(self) -> Dict[str, Any]: + data = self.summary() + data["batch_id"] = self.batch_id + return data diff --git a/scripts/service/example_adapter/sim_rpa.py b/scripts/service/example_adapter/sim_rpa.py index 26b1555..f17fc33 100644 --- a/scripts/service/example_adapter/sim_rpa.py +++ b/scripts/service/example_adapter/sim_rpa.py @@ -50,14 +50,13 @@ class SimRpaAdapter(AdapterBase): batch_id=batch_id, title="技能仿真演示", ) as video: - video.step("正在打开目标页面") + video.add_step("正在打开目标页面") try: async with async_playwright() as pw: context = await launch_persistent_browser( pw, profile_dir=profile_dir, headless=config.get_bool("OPENCLAW_BROWSER_HEADLESS"), - record_video_dir=video.record_video_dir, ) page = context.pages[0] if context.pages else await context.new_page() await page.goto(url, wait_until="domcontentloaded", timeout=60_000)