refactor: ffmpeg desktop capture replaces Playwright recording

Co-authored-by: Cursor <cursoragent@cursor.com>

src/jiangchang_skill_core/rpa/__init__.py

@@ -9,6 +9,11 @@ try:
 except ImportError:
     launch_persistent_browser = None  # type: ignore[misc, assignment]
 
+try:
+    from .video_session import RpaVideoSession
+except ImportError:
+    RpaVideoSession = None  # type: ignore[misc, assignment]
+
 __all__ = [
     "anti_detect",
     "artifacts",
@@ -16,6 +21,7 @@ __all__ = [
     "human_login",
     "stealth",
     "launch_persistent_browser",
+    "RpaVideoSession",
     # re-export common anti_detect helpers
     "random_delay",
     "human_delay_short",

src/jiangchang_skill_core/rpa/browser.py

@@ -1,4 +1,4 @@
-"""统一 persistent context 启动封装。"""
+"""统一 persistent context 启动封装（仅浏览器自动化，不负责录屏）。"""
 
 from __future__ import annotations
 
@@ -24,12 +24,11 @@ async def launch_persistent_browser(
     executable_path: str | None = None,
     headless: bool | None = None,
     extra_args: list[str] | None = None,
-    record_video_dir: str | None = None,
 ) -> "BrowserContext":
     """用 stealth 参数启动 persistent context + 注入 STEALTH_INIT_SCRIPT。
 
     headless=None 时读 OPENCLAW_BROWSER_HEADLESS。
-    record_video_dir 非空则开录屏。
+    录屏由 RpaVideoSession（ffmpeg）负责，本函数不向 Playwright 传递任何录屏参数。
     """
     if headless is None:
         v = (os.environ.get("OPENCLAW_BROWSER_HEADLESS") or "0").strip().lower()
@@ -52,8 +51,6 @@ async def launch_persistent_browser(
     )
     if ignore is not None:
         launch_kwargs["ignore_default_args"] = ignore
-    if record_video_dir:
-        launch_kwargs["record_video_dir"] = record_video_dir
 
     context = await playwright.chromium.launch_persistent_context(**launch_kwargs)
     if stealth_enabled():

src/jiangchang_skill_core/rpa/video_session.py

@@ -0,0 +1,420 @@
+"""RPA 运行录屏会话：ffmpeg 桌面录制 → 字幕 + 背景音乐 → 最终 MP4。
+
+Playwright 仅用于浏览器自动化，不负责录屏。
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import random
+import shutil
+import subprocess
+import sys
+import time
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from .. import config
+
+_JIANGCHANG_SUBTITLE_STYLE = (
+    "FontName=Arial,FontSize=14,"
+    "PrimaryColour=&H00FFFFFF,OutlineColour=&H00000000,"
+    "Outline=2,Shadow=1,Alignment=2"
+)
+
+_DEFAULT_MUSIC_ROOT = os.environ.get(
+    "MEDIA_ASSETS_ROOT",
+    r"D:\OpenClaw\client-commons\media-assets",
+)
+
+_CAPTURE_FRAMERATE = 15
+
+
+def _record_video_enabled() -> bool:
+    return config.get_bool("OPENCLAW_RECORD_VIDEO", default=False)
+
+
+def _resolve_music_file() -> Optional[Path]:
+    music_dir = Path(_DEFAULT_MUSIC_ROOT) / "music"
+    if not music_dir.is_dir():
+        return None
+    files = list(music_dir.rglob("*.mp3"))
+    if not files:
+        return None
+    return random.choice(files)
+
+
+@dataclass
+class _StepEntry:
+    start: float
+    text: str
+    duration: float = 4.0
+
+
+def _generate_srt(entries: List[_StepEntry], output_path: str) -> None:
+    MIN_DURATION = 1.0
+    GAP = 0.1
+    steps = sorted(entries, key=lambda e: e.start)
+    if not steps:
+        steps = [_StepEntry(0.0, "任务执行中", 4.0)]
+
+    for i in range(len(steps) - 1):
+        cur, nxt = steps[i], steps[i + 1]
+        max_end = nxt.start - GAP
+        new_dur = max_end - cur.start
+        if new_dur < MIN_DURATION:
+            cur.duration = MIN_DURATION
+            nxt.start = cur.start + MIN_DURATION + GAP
+        else:
+            cur.duration = min(cur.duration, new_dur)
+
+    def _fmt(s: float) -> str:
+        h = int(s // 3600)
+        m = int((s % 3600) // 60)
+        sec = int(s % 60)
+        ms = int((s % 1) * 1000)
+        return f"{h:02d}:{m:02d}:{sec:02d},{ms:03d}"
+
+    Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+    with open(output_path, "w", encoding="utf-8") as f:
+        for i, e in enumerate(steps, 1):
+            f.write(f"{i}\n")
+            f.write(f"{_fmt(e.start)} --> {_fmt(e.start + e.duration)}\n")
+            f.write(f"{e.text}\n\n")
+
+
+def _escape_srt_path(path: str) -> str:
+    return path.replace("\\", "/").replace(":", "\\:")
+
+
+def _run_ffmpeg(cmd: List[str]) -> tuple[bool, str]:
+    try:
+        result = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+            encoding="utf-8",
+            errors="replace",
+        )
+    except FileNotFoundError:
+        return False, "ffmpeg not found"
+    if result.returncode != 0:
+        err = (result.stderr or result.stdout or "").strip()
+        return False, err[:500] if err else f"exit {result.returncode}"
+    return True, ""
+
+
+def _build_desktop_record_cmd(capture_path: str) -> Optional[List[str]]:
+    """Windows gdigrab 全桌面录制；非 Windows 返回 None。"""
+    if sys.platform != "win32":
+        return None
+    return [
+        "ffmpeg",
+        "-y",
+        "-f",
+        "gdigrab",
+        "-framerate",
+        str(_CAPTURE_FRAMERATE),
+        "-i",
+        "desktop",
+        "-c:v",
+        "libx264",
+        "-preset",
+        "ultrafast",
+        "-pix_fmt",
+        "yuv420p",
+        capture_path,
+    ]
+
+
+def _compose_capture_mp4(
+    capture_path: Path,
+    srt_path: Path,
+    output_path: Path,
+    *,
+    music_file: Optional[Path] = None,
+    music_volume: float = 0.15,
+) -> tuple[bool, str]:
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    srt_esc = _escape_srt_path(str(srt_path.resolve()))
+    subtitle_filter = (
+        f"subtitles='{srt_esc}'"
+        f":force_style='{_JIANGCHANG_SUBTITLE_STYLE}'"
+    )
+
+    if music_file and music_file.is_file():
+        filter_complex = (
+            f"[0:v]{subtitle_filter}[vout];"
+            f"[1:a]volume={music_volume},apad[aout]"
+        )
+        cmd = [
+            "ffmpeg",
+            "-y",
+            "-i",
+            str(capture_path),
+            "-i",
+            str(music_file),
+            "-filter_complex",
+            filter_complex,
+            "-map",
+            "[vout]",
+            "-map",
+            "[aout]",
+            "-c:v",
+            "libx264",
+            "-preset",
+            "fast",
+            "-crf",
+            "23",
+            "-c:a",
+            "aac",
+            "-b:a",
+            "128k",
+            "-shortest",
+            "-pix_fmt",
+            "yuv420p",
+            str(output_path),
+        ]
+    else:
+        cmd = [
+            "ffmpeg",
+            "-y",
+            "-i",
+            str(capture_path),
+            "-vf",
+            subtitle_filter,
+            "-c:v",
+            "libx264",
+            "-preset",
+            "fast",
+            "-crf",
+            "23",
+            "-c:a",
+            "aac",
+            "-b:a",
+            "128k",
+            "-pix_fmt",
+            "yuv420p",
+            str(output_path),
+        ]
+    return _run_ffmpeg(cmd)
+
+
+class RpaVideoSession:
+    """Service/RPA 层统一 ffmpeg 桌面录屏；OPENCLAW_RECORD_VIDEO=1 时启用成片流程。"""
+
+    def __init__(
+        self,
+        *,
+        skill_slug: str,
+        skill_data_dir: str,
+        batch_id: str,
+        title: str = "",
+    ) -> None:
+        self.skill_slug = skill_slug
+        self.skill_data_dir = os.path.abspath(skill_data_dir)
+        self.batch_id = batch_id
+        self.title = title
+        self.enabled = _record_video_enabled()
+        self.warnings: List[str] = []
+        self.output_video_path: Optional[str] = None
+        self.capture_path: Optional[str] = None
+        # 已废弃：勿用于 Playwright；始终为 None。
+        self.record_video_dir: Optional[str] = None
+        self.artifacts: Dict[str, Any] = {}
+
+        ts = datetime.now().strftime("%Y%m%d_%H%M%S")
+        safe_batch = "".join(c if c.isalnum() or c in "-_" else "_" for c in batch_id)
+        self._video_stem = f"{skill_slug}_{ts}_{safe_batch}"
+
+        self._videos_dir = os.path.join(self.skill_data_dir, "videos")
+        self._artifact_root = os.path.join(
+            self.skill_data_dir, "rpa-artifacts", batch_id
+        )
+        self._subtitles_dir = os.path.join(self._artifact_root, "subtitles")
+        self._logs_dir = os.path.join(self._artifact_root, "logs")
+        self._capture_path = os.path.join(self._artifact_root, "capture.mp4")
+        self._srt_path = os.path.join(self._subtitles_dir, f"{self._video_stem}.srt")
+        self._planned_output = os.path.join(self._videos_dir, f"{self._video_stem}.mp4")
+
+        self._steps: List[_StepEntry] = []
+        self._t0: Optional[float] = None
+        self._ffmpeg_proc: Optional[subprocess.Popen] = None
+
+        if self.enabled:
+            os.makedirs(self._videos_dir, exist_ok=True)
+            os.makedirs(self._subtitles_dir, exist_ok=True)
+            os.makedirs(self._logs_dir, exist_ok=True)
+            os.makedirs(self._artifact_root, exist_ok=True)
+
+    def step(self, text: str, *, duration: float = 4.0) -> None:
+        self.add_step(text, duration=duration)
+
+    def add_step(self, text: str, *, duration: float = 4.0) -> None:
+        """记录一步骤字幕（相对录屏开始时间）。"""
+        if not self.enabled:
+            return
+        if self._t0 is None:
+            self._t0 = time.monotonic()
+        elapsed = time.monotonic() - self._t0
+        self._steps.append(_StepEntry(start=elapsed, text=text.strip(), duration=duration))
+
+    def _start_ffmpeg_capture(self) -> None:
+        if not self.enabled:
+            return
+        if not shutil.which("ffmpeg"):
+            self.warnings.append("ffmpeg_not_found")
+            return
+
+        cmd = _build_desktop_record_cmd(self._capture_path)
+        if cmd is None:
+            self.warnings.append(
+                "ffmpeg_record_start_failed: desktop capture only supported on Windows"
+            )
+            return
+
+        try:
+            self._ffmpeg_proc = subprocess.Popen(
+                cmd,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+            self.capture_path = self._capture_path
+        except Exception as exc:
+            self.warnings.append(f"ffmpeg_record_start_failed: {exc}")
+            self._ffmpeg_proc = None
+
+    def _stop_ffmpeg_capture(self) -> None:
+        proc = self._ffmpeg_proc
+        if proc is None:
+            return
+        if proc.poll() is None:
+            try:
+                if proc.stdin:
+                    proc.stdin.write(b"q")
+                    proc.stdin.flush()
+            except Exception:
+                try:
+                    proc.terminate()
+                except Exception:
+                    pass
+            try:
+                proc.wait(timeout=20)
+            except subprocess.TimeoutExpired:
+                try:
+                    proc.kill()
+                except Exception:
+                    pass
+                try:
+                    proc.wait(timeout=5)
+                except Exception:
+                    pass
+        self._ffmpeg_proc = None
+        await_sleep = 0.3
+        time.sleep(await_sleep)
+
+    async def __aenter__(self) -> "RpaVideoSession":
+        if self.enabled:
+            self._t0 = time.monotonic()
+            if self.title:
+                self.add_step(self.title, duration=3.0)
+            self._start_ffmpeg_capture()
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb) -> None:
+        if self.enabled:
+            self._stop_ffmpeg_capture()
+            await self.finalize()
+
+    async def finalize(self) -> None:
+        """停止录制后合成最终 MP4；失败只记 warning，不抛异常。"""
+        if not self.enabled:
+            self.artifacts = self.summary()
+            return
+
+        if not self._steps and self.title:
+            self.add_step(self.title)
+
+        try:
+            capture = Path(self._capture_path)
+            if not capture.is_file() or capture.stat().st_size <= 0:
+                self.warnings.append("ffmpeg_capture_missing")
+                self.artifacts = self.summary()
+                return
+
+            self.capture_path = str(capture.resolve())
+
+            entries = list(self._steps)
+            if entries:
+                entries[-1].duration = max(entries[-1].duration, 5.0)
+            try:
+                _generate_srt(entries, self._srt_path)
+            except Exception as exc:
+                self.warnings.append(f"subtitle_write_failed: {exc}")
+
+            music = _resolve_music_file()
+            if music is None:
+                self.warnings.append("background_music_missing")
+
+            srt_file = Path(self._srt_path)
+            use_srt = srt_file.is_file() and srt_file.stat().st_size > 0
+
+            ok, err = _compose_capture_mp4(
+                capture,
+                srt_file if use_srt else Path(self._srt_path),
+                Path(self._planned_output),
+                music_file=music,
+            )
+            if not ok and music is not None:
+                self.warnings.append(f"ffmpeg_with_music_failed: {err}")
+                ok, err = _compose_capture_mp4(
+                    capture,
+                    srt_file if use_srt else Path(self._srt_path),
+                    Path(self._planned_output),
+                    music_file=None,
+                )
+            if not ok and not use_srt:
+                ok, err = _run_ffmpeg(
+                    [
+                        "ffmpeg",
+                        "-y",
+                        "-i",
+                        str(capture),
+                        "-c:v",
+                        "libx264",
+                        "-preset",
+                        "fast",
+                        "-crf",
+                        "23",
+                        "-pix_fmt",
+                        "yuv420p",
+                        str(self._planned_output),
+                    ]
+                )
+            if not ok:
+                self.warnings.append(f"ffmpeg_compose_failed: {err}")
+                self.artifacts = self.summary()
+                return
+
+            self.output_video_path = str(Path(self._planned_output).resolve())
+        except Exception as exc:
+            self.warnings.append(f"video_finalize_error: {exc}")
+
+        self.artifacts = self.summary()
+
+    def summary(self) -> Dict[str, Any]:
+        return {
+            "enabled": self.enabled,
+            "path": self.output_video_path,
+            "capture_path": self.capture_path,
+            "warnings": list(self.warnings),
+        }
+
+    def to_artifact_dict(self) -> Dict[str, Any]:
+        data = self.summary()
+        data["batch_id"] = self.batch_id
+        return data

tests/test_video_session.py

@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*-
+"""RpaVideoSession：ffmpeg 桌面录屏，无 Playwright record_video_dir。"""
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+import tempfile
+import unittest
+from unittest.mock import MagicMock, patch
+
+from jiangchang_skill_core import config
+from jiangchang_skill_core.rpa.video_session import RpaVideoSession
+
+
+class TestRpaVideoSession(unittest.TestCase):
+    def setUp(self) -> None:
+        config.reset_cache()
+
+    def test_disabled_is_noop(self) -> None:
+        old = os.environ.pop("OPENCLAW_RECORD_VIDEO", None)
+        try:
+            with tempfile.TemporaryDirectory() as tmp:
+                config.reset_cache()
+                session = RpaVideoSession(
+                    skill_slug="receive-order",
+                    skill_data_dir=tmp,
+                    batch_id="batch_test",
+                )
+                self.assertFalse(session.enabled)
+                self.assertIsNone(session.record_video_dir)
+                self.assertIsNone(session.output_video_path)
+                self.assertIsNone(session.capture_path)
+                summ = session.summary()
+                self.assertFalse(summ["enabled"])
+                self.assertIsNone(summ["path"])
+        finally:
+            if old is not None:
+                os.environ["OPENCLAW_RECORD_VIDEO"] = old
+            config.reset_cache()
+
+    def test_enabled_paths_no_raw_video(self) -> None:
+        os.environ["OPENCLAW_RECORD_VIDEO"] = "1"
+        try:
+            with tempfile.TemporaryDirectory() as tmp:
+                config.reset_cache()
+                session = RpaVideoSession(
+                    skill_slug="receive-order",
+                    skill_data_dir=tmp,
+                    batch_id="exmail_20260602_120000",
+                    title="接收订单",
+                )
+                self.assertTrue(session.enabled)
+                self.assertIsNone(session.record_video_dir)
+                self.assertTrue(
+                    session._capture_path.replace("\\", "/").endswith(
+                        "/rpa-artifacts/exmail_20260602_120000/capture.mp4"
+                    )
+                )
+                self.assertTrue(
+                    session._planned_output.replace("\\", "/").startswith(
+                        tmp.replace("\\", "/") + "/videos/"
+                    )
+                )
+        finally:
+            os.environ.pop("OPENCLAW_RECORD_VIDEO", None)
+            config.reset_cache()
+
+    @unittest.skipUnless(sys.platform == "win32", "gdigrab desktop capture")
+    @patch("jiangchang_skill_core.rpa.video_session.shutil.which", return_value="ffmpeg")
+    @patch("jiangchang_skill_core.rpa.video_session.subprocess.Popen")
+    def test_enter_starts_ffmpeg_gdigrab(self, mock_popen: MagicMock, _which: MagicMock) -> None:
+        proc = MagicMock()
+        proc.poll.return_value = None
+        proc.stdin = MagicMock()
+        mock_popen.return_value = proc
+
+        os.environ["OPENCLAW_RECORD_VIDEO"] = "1"
+        try:
+            with tempfile.TemporaryDirectory() as tmp:
+                config.reset_cache()
+
+                async def _run() -> None:
+                    async with RpaVideoSession(
+                        skill_slug="receive-order",
+                        skill_data_dir=tmp,
+                        batch_id="batch_ff",
+                    ) as video:
+                        video.add_step("测试步骤")
+                        with patch.object(video, "finalize", return_value=None):
+                            pass
+
+                asyncio.run(_run())
+                self.assertTrue(mock_popen.called)
+                cmd = mock_popen.call_args[0][0]
+                self.assertIn("ffmpeg", cmd)
+                self.assertIn("gdigrab", cmd)
+                self.assertIn("desktop", cmd)
+        finally:
+            os.environ.pop("OPENCLAW_RECORD_VIDEO", None)
+            config.reset_cache()
+
+    @patch("jiangchang_skill_core.rpa.video_session.shutil.which", return_value=None)
+    def test_ffmpeg_not_found_warning(self, _which: MagicMock) -> None:
+        os.environ["OPENCLAW_RECORD_VIDEO"] = "1"
+        try:
+            with tempfile.TemporaryDirectory() as tmp:
+                config.reset_cache()
+
+                async def _run() -> None:
+                    async with RpaVideoSession(
+                        skill_slug="receive-order",
+                        skill_data_dir=tmp,
+                        batch_id="batch_no_ff",
+                    ):
+                        pass
+
+                asyncio.run(_run())
+                session = RpaVideoSession(
+                    skill_slug="receive-order",
+                    skill_data_dir=tmp,
+                    batch_id="batch_no_ff",
+                )
+                session._start_ffmpeg_capture()
+                self.assertIn("ffmpeg_not_found", session.warnings)
+        finally:
+            os.environ.pop("OPENCLAW_RECORD_VIDEO", None)
+            config.reset_cache()
+
+    def test_finalize_missing_capture_warning(self) -> None:
+        os.environ["OPENCLAW_RECORD_VIDEO"] = "1"
+        try:
+            with tempfile.TemporaryDirectory() as tmp:
+                config.reset_cache()
+                session = RpaVideoSession(
+                    skill_slug="receive-order",
+                    skill_data_dir=tmp,
+                    batch_id="batch_warn",
+                )
+                asyncio.run(session.finalize())
+                self.assertIn("ffmpeg_capture_missing", session.warnings)
+                self.assertIsNone(session.output_video_path)
+                summ = session.summary()
+                self.assertIsNone(summ["path"])
+        finally:
+            os.environ.pop("OPENCLAW_RECORD_VIDEO", None)
+            config.reset_cache()
+
+
+if __name__ == "__main__":
+    unittest.main()