refactor: ffmpeg desktop capture replaces Playwright recording

Co-authored-by: Cursor <cursoragent@cursor.com>

src/jiangchang_skill_core/rpa/browser.py

@@ -1,4 +1,4 @@
-"""统一 persistent context 启动封装。"""
+"""统一 persistent context 启动封装（仅浏览器自动化，不负责录屏）。"""
 
 from __future__ import annotations
 
@@ -24,12 +24,11 @@ async def launch_persistent_browser(
     executable_path: str | None = None,
     headless: bool | None = None,
     extra_args: list[str] | None = None,
-    record_video_dir: str | None = None,
 ) -> "BrowserContext":
     """用 stealth 参数启动 persistent context + 注入 STEALTH_INIT_SCRIPT。
 
     headless=None 时读 OPENCLAW_BROWSER_HEADLESS。
-    record_video_dir 非空则开录屏。
+    录屏由 RpaVideoSession（ffmpeg）负责，本函数不向 Playwright 传递任何录屏参数。
     """
     if headless is None:
         v = (os.environ.get("OPENCLAW_BROWSER_HEADLESS") or "0").strip().lower()
@@ -52,8 +51,6 @@ async def launch_persistent_browser(
     )
     if ignore is not None:
         launch_kwargs["ignore_default_args"] = ignore
-    if record_video_dir:
-        launch_kwargs["record_video_dir"] = record_video_dir
 
     context = await playwright.chromium.launch_persistent_context(**launch_kwargs)
     if stealth_enabled():

src/jiangchang_skill_core/rpa/video_session.py

@@ -1,10 +1,14 @@
-"""RPA 运行录屏会话：Playwright raw video → 字幕 + 背景音乐 → 最终 MP4。"""
+"""RPA 运行录屏会话：ffmpeg 桌面录制 → 字幕 + 背景音乐 → 最终 MP4。
+
+Playwright 仅用于浏览器自动化，不负责录屏。
+"""
 
 from __future__ import annotations
 
 import asyncio
 import os
 import random
+import shutil
 import subprocess
 import sys
 import time
@@ -15,7 +19,6 @@ from typing import Any, Dict, List, Optional
 
 from .. import config
 
-# 与 screencast/composer.py 保持一致的字幕样式
 _JIANGCHANG_SUBTITLE_STYLE = (
     "FontName=Arial,FontSize=14,"
     "PrimaryColour=&H00FFFFFF,OutlineColour=&H00000000,"
@@ -27,6 +30,8 @@ _DEFAULT_MUSIC_ROOT = os.environ.get(
     r"D:\OpenClaw\client-commons\media-assets",
 )
 
+_CAPTURE_FRAMERATE = 15
+
 
 def _record_video_enabled() -> bool:
     return config.get_bool("OPENCLAW_RECORD_VIDEO", default=False)
@@ -50,7 +55,6 @@ class _StepEntry:
 
 
 def _generate_srt(entries: List[_StepEntry], output_path: str) -> None:
-    """根据步骤时间线生成 SRT（防重叠）。"""
     MIN_DURATION = 1.0
     GAP = 0.1
     steps = sorted(entries, key=lambda e: e.start)
@@ -86,16 +90,6 @@ def _escape_srt_path(path: str) -> str:
     return path.replace("\\", "/").replace(":", "\\:")
 
 
-def _find_raw_webm(raw_dir: str) -> Optional[Path]:
-    root = Path(raw_dir)
-    if not root.is_dir():
-        return None
-    webms = list(root.rglob("*.webm"))
-    if not webms:
-        return None
-    return max(webms, key=lambda p: p.stat().st_size)
-
-
 def _run_ffmpeg(cmd: List[str]) -> tuple[bool, str]:
     try:
         result = subprocess.run(
@@ -113,8 +107,31 @@ def _run_ffmpeg(cmd: List[str]) -> tuple[bool, str]:
     return True, ""
 
 
-def _compose_webm_mp4(
+def _build_desktop_record_cmd(capture_path: str) -> Optional[List[str]]:
-    webm_path: Path,
+    """Windows gdigrab 全桌面录制；非 Windows 返回 None。"""
+    if sys.platform != "win32":
+        return None
+    return [
+        "ffmpeg",
+        "-y",
+        "-f",
+        "gdigrab",
+        "-framerate",
+        str(_CAPTURE_FRAMERATE),
+        "-i",
+        "desktop",
+        "-c:v",
+        "libx264",
+        "-preset",
+        "ultrafast",
+        "-pix_fmt",
+        "yuv420p",
+        capture_path,
+    ]
+
+
+def _compose_capture_mp4(
+    capture_path: Path,
     srt_path: Path,
     output_path: Path,
     *,
@@ -134,39 +151,60 @@ def _compose_webm_mp4(
             f"[1:a]volume={music_volume},apad[aout]"
         )
         cmd = [
-            "ffmpeg", "-y",
+            "ffmpeg",
-            "-i", str(webm_path),
+            "-y",
-            "-i", str(music_file),
+            "-i",
-            "-filter_complex", filter_complex,
+            str(capture_path),
-            "-map", "[vout]",
+            "-i",
-            "-map", "[aout]",
+            str(music_file),
-            "-c:v", "libx264",
+            "-filter_complex",
-            "-preset", "fast",
+            filter_complex,
-            "-crf", "23",
+            "-map",
-            "-c:a", "aac",
+            "[vout]",
-            "-b:a", "128k",
+            "-map",
+            "[aout]",
+            "-c:v",
+            "libx264",
+            "-preset",
+            "fast",
+            "-crf",
+            "23",
+            "-c:a",
+            "aac",
+            "-b:a",
+            "128k",
             "-shortest",
-            "-pix_fmt", "yuv420p",
+            "-pix_fmt",
+            "yuv420p",
             str(output_path),
         ]
     else:
         cmd = [
-            "ffmpeg", "-y",
+            "ffmpeg",
-            "-i", str(webm_path),
+            "-y",
-            "-vf", subtitle_filter,
+            "-i",
-            "-c:v", "libx264",
+            str(capture_path),
-            "-preset", "fast",
+            "-vf",
-            "-crf", "23",
+            subtitle_filter,
-            "-c:a", "aac",
+            "-c:v",
-            "-b:a", "128k",
+            "libx264",
-            "-pix_fmt", "yuv420p",
+            "-preset",
+            "fast",
+            "-crf",
+            "23",
+            "-c:a",
+            "aac",
+            "-b:a",
+            "128k",
+            "-pix_fmt",
+            "yuv420p",
             str(output_path),
         ]
     return _run_ffmpeg(cmd)
 
 
 class RpaVideoSession:
-    """Service/RPA 层统一录屏会话；OPENCLAW_RECORD_VIDEO=1 时启用完整成片流程。"""
+    """Service/RPA 层统一 ffmpeg 桌面录屏；OPENCLAW_RECORD_VIDEO=1 时启用成片流程。"""
 
     def __init__(
         self,
@@ -183,6 +221,8 @@ class RpaVideoSession:
         self.enabled = _record_video_enabled()
         self.warnings: List[str] = []
         self.output_video_path: Optional[str] = None
+        self.capture_path: Optional[str] = None
+        # 已废弃：勿用于 Playwright；始终为 None。
         self.record_video_dir: Optional[str] = None
         self.artifacts: Dict[str, Any] = {}
 
@@ -194,24 +234,27 @@ class RpaVideoSession:
         self._artifact_root = os.path.join(
             self.skill_data_dir, "rpa-artifacts", batch_id
         )
-        self._raw_video_dir = os.path.join(self._artifact_root, "raw-video")
         self._subtitles_dir = os.path.join(self._artifact_root, "subtitles")
         self._logs_dir = os.path.join(self._artifact_root, "logs")
+        self._capture_path = os.path.join(self._artifact_root, "capture.mp4")
         self._srt_path = os.path.join(self._subtitles_dir, f"{self._video_stem}.srt")
         self._planned_output = os.path.join(self._videos_dir, f"{self._video_stem}.mp4")
 
         self._steps: List[_StepEntry] = []
         self._t0: Optional[float] = None
+        self._ffmpeg_proc: Optional[subprocess.Popen] = None
 
         if self.enabled:
             os.makedirs(self._videos_dir, exist_ok=True)
-            os.makedirs(self._raw_video_dir, exist_ok=True)
             os.makedirs(self._subtitles_dir, exist_ok=True)
             os.makedirs(self._logs_dir, exist_ok=True)
-            self.record_video_dir = self._raw_video_dir
+            os.makedirs(self._artifact_root, exist_ok=True)
 
     def step(self, text: str, *, duration: float = 4.0) -> None:
-        """记录一步骤字幕（中文说明）。"""
+        self.add_step(text, duration=duration)
+
+    def add_step(self, text: str, *, duration: float = 4.0) -> None:
+        """记录一步骤字幕（相对录屏开始时间）。"""
         if not self.enabled:
             return
         if self._t0 is None:
@@ -219,71 +262,159 @@ class RpaVideoSession:
         elapsed = time.monotonic() - self._t0
         self._steps.append(_StepEntry(start=elapsed, text=text.strip(), duration=duration))
 
+    def _start_ffmpeg_capture(self) -> None:
+        if not self.enabled:
+            return
+        if not shutil.which("ffmpeg"):
+            self.warnings.append("ffmpeg_not_found")
+            return
+
+        cmd = _build_desktop_record_cmd(self._capture_path)
+        if cmd is None:
+            self.warnings.append(
+                "ffmpeg_record_start_failed: desktop capture only supported on Windows"
+            )
+            return
+
+        try:
+            self._ffmpeg_proc = subprocess.Popen(
+                cmd,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+            self.capture_path = self._capture_path
+        except Exception as exc:
+            self.warnings.append(f"ffmpeg_record_start_failed: {exc}")
+            self._ffmpeg_proc = None
+
+    def _stop_ffmpeg_capture(self) -> None:
+        proc = self._ffmpeg_proc
+        if proc is None:
+            return
+        if proc.poll() is None:
+            try:
+                if proc.stdin:
+                    proc.stdin.write(b"q")
+                    proc.stdin.flush()
+            except Exception:
+                try:
+                    proc.terminate()
+                except Exception:
+                    pass
+            try:
+                proc.wait(timeout=20)
+            except subprocess.TimeoutExpired:
+                try:
+                    proc.kill()
+                except Exception:
+                    pass
+                try:
+                    proc.wait(timeout=5)
+                except Exception:
+                    pass
+        self._ffmpeg_proc = None
+        await_sleep = 0.3
+        time.sleep(await_sleep)
+
     async def __aenter__(self) -> "RpaVideoSession":
         if self.enabled:
             self._t0 = time.monotonic()
             if self.title:
-                self.step(self.title, duration=3.0)
+                self.add_step(self.title, duration=3.0)
+            self._start_ffmpeg_capture()
         return self
 
     async def __aexit__(self, exc_type, exc, tb) -> None:
         if self.enabled:
+            self._stop_ffmpeg_capture()
             await self.finalize()
 
     async def finalize(self) -> None:
-        """在 Playwright context 已 close 后合成最终 MP4；失败只记 warning。"""
+        """停止录制后合成最终 MP4；失败只记 warning，不抛异常。"""
         if not self.enabled:
+            self.artifacts = self.summary()
             return
 
         if not self._steps and self.title:
-            self.step(self.title)
+            self.add_step(self.title)
 
         try:
-            await asyncio.sleep(0.5)
+            capture = Path(self._capture_path)
-            webm = _find_raw_webm(self._raw_video_dir)
+            if not capture.is_file() or capture.stat().st_size <= 0:
-            if webm is None:
+                self.warnings.append("ffmpeg_capture_missing")
-                self.warnings.append("no_playwright_raw_video")
+                self.artifacts = self.summary()
-                self.artifacts = self.to_artifact_dict()
                 return
 
+            self.capture_path = str(capture.resolve())
+
             entries = list(self._steps)
             if entries:
                 entries[-1].duration = max(entries[-1].duration, 5.0)
-            _generate_srt(entries, self._srt_path)
+            try:
+                _generate_srt(entries, self._srt_path)
+            except Exception as exc:
+                self.warnings.append(f"subtitle_write_failed: {exc}")
 
             music = _resolve_music_file()
             if music is None:
                 self.warnings.append("background_music_missing")
 
-            ok, err = _compose_webm_mp4(
+            srt_file = Path(self._srt_path)
-                webm,
+            use_srt = srt_file.is_file() and srt_file.stat().st_size > 0
-                Path(self._srt_path),
+
+            ok, err = _compose_capture_mp4(
+                capture,
+                srt_file if use_srt else Path(self._srt_path),
                 Path(self._planned_output),
                 music_file=music,
             )
             if not ok and music is not None:
                 self.warnings.append(f"ffmpeg_with_music_failed: {err}")
-                ok, err = _compose_webm_mp4(
+                ok, err = _compose_capture_mp4(
-                    webm,
+                    capture,
-                    Path(self._srt_path),
+                    srt_file if use_srt else Path(self._srt_path),
                     Path(self._planned_output),
                     music_file=None,
                 )
+            if not ok and not use_srt:
+                ok, err = _run_ffmpeg(
+                    [
+                        "ffmpeg",
+                        "-y",
+                        "-i",
+                        str(capture),
+                        "-c:v",
+                        "libx264",
+                        "-preset",
+                        "fast",
+                        "-crf",
+                        "23",
+                        "-pix_fmt",
+                        "yuv420p",
+                        str(self._planned_output),
+                    ]
+                )
             if not ok:
                 self.warnings.append(f"ffmpeg_compose_failed: {err}")
-                self.artifacts = self.to_artifact_dict()
+                self.artifacts = self.summary()
                 return
 
             self.output_video_path = str(Path(self._planned_output).resolve())
-            self.artifacts = self.to_artifact_dict()
         except Exception as exc:
             self.warnings.append(f"video_finalize_error: {exc}")
-            self.artifacts = self.to_artifact_dict()
 
-    def to_artifact_dict(self) -> Dict[str, Any]:
+        self.artifacts = self.summary()
+
+    def summary(self) -> Dict[str, Any]:
         return {
             "enabled": self.enabled,
             "path": self.output_video_path,
+            "capture_path": self.capture_path,
             "warnings": list(self.warnings),
-            "batch_id": self.batch_id,
         }
+
+    def to_artifact_dict(self) -> Dict[str, Any]:
+        data = self.summary()
+        data["batch_id"] = self.batch_id
+        return data

tests/test_video_session.py

@@ -1,10 +1,13 @@
 # -*- coding: utf-8 -*-
-"""RpaVideoSession 路径与 no-op 行为。"""
+"""RpaVideoSession：ffmpeg 桌面录屏，无 Playwright record_video_dir。"""
 from __future__ import annotations
 
+import asyncio
 import os
+import sys
 import tempfile
 import unittest
+from unittest.mock import MagicMock, patch
 
 from jiangchang_skill_core import config
 from jiangchang_skill_core.rpa.video_session import RpaVideoSession
@@ -27,12 +30,16 @@ class TestRpaVideoSession(unittest.TestCase):
                 self.assertFalse(session.enabled)
                 self.assertIsNone(session.record_video_dir)
                 self.assertIsNone(session.output_video_path)
+                self.assertIsNone(session.capture_path)
+                summ = session.summary()
+                self.assertFalse(summ["enabled"])
+                self.assertIsNone(summ["path"])
         finally:
             if old is not None:
                 os.environ["OPENCLAW_RECORD_VIDEO"] = old
             config.reset_cache()
 
-    def test_enabled_paths(self) -> None:
+    def test_enabled_paths_no_raw_video(self) -> None:
         os.environ["OPENCLAW_RECORD_VIDEO"] = "1"
         try:
             with tempfile.TemporaryDirectory() as tmp:
@@ -44,9 +51,10 @@ class TestRpaVideoSession(unittest.TestCase):
                     title="接收订单",
                 )
                 self.assertTrue(session.enabled)
+                self.assertIsNone(session.record_video_dir)
                 self.assertTrue(
-                    session.record_video_dir.replace("\\", "/").endswith(
+                    session._capture_path.replace("\\", "/").endswith(
-                        "/rpa-artifacts/exmail_20260602_120000/raw-video"
+                        "/rpa-artifacts/exmail_20260602_120000/capture.mp4"
                     )
                 )
                 self.assertTrue(
@@ -54,15 +62,72 @@ class TestRpaVideoSession(unittest.TestCase):
                         tmp.replace("\\", "/") + "/videos/"
                     )
                 )
-                self.assertNotIn(
-                    "rpa-artifacts",
-                    session._planned_output.replace("\\", "/").split("/videos/")[-1],
-                )
         finally:
             os.environ.pop("OPENCLAW_RECORD_VIDEO", None)
             config.reset_cache()
 
-    def test_finalize_missing_webm_only_warning(self) -> None:
+    @unittest.skipUnless(sys.platform == "win32", "gdigrab desktop capture")
+    @patch("jiangchang_skill_core.rpa.video_session.shutil.which", return_value="ffmpeg")
+    @patch("jiangchang_skill_core.rpa.video_session.subprocess.Popen")
+    def test_enter_starts_ffmpeg_gdigrab(self, mock_popen: MagicMock, _which: MagicMock) -> None:
+        proc = MagicMock()
+        proc.poll.return_value = None
+        proc.stdin = MagicMock()
+        mock_popen.return_value = proc
+
+        os.environ["OPENCLAW_RECORD_VIDEO"] = "1"
+        try:
+            with tempfile.TemporaryDirectory() as tmp:
+                config.reset_cache()
+
+                async def _run() -> None:
+                    async with RpaVideoSession(
+                        skill_slug="receive-order",
+                        skill_data_dir=tmp,
+                        batch_id="batch_ff",
+                    ) as video:
+                        video.add_step("测试步骤")
+                        with patch.object(video, "finalize", return_value=None):
+                            pass
+
+                asyncio.run(_run())
+                self.assertTrue(mock_popen.called)
+                cmd = mock_popen.call_args[0][0]
+                self.assertIn("ffmpeg", cmd)
+                self.assertIn("gdigrab", cmd)
+                self.assertIn("desktop", cmd)
+        finally:
+            os.environ.pop("OPENCLAW_RECORD_VIDEO", None)
+            config.reset_cache()
+
+    @patch("jiangchang_skill_core.rpa.video_session.shutil.which", return_value=None)
+    def test_ffmpeg_not_found_warning(self, _which: MagicMock) -> None:
+        os.environ["OPENCLAW_RECORD_VIDEO"] = "1"
+        try:
+            with tempfile.TemporaryDirectory() as tmp:
+                config.reset_cache()
+
+                async def _run() -> None:
+                    async with RpaVideoSession(
+                        skill_slug="receive-order",
+                        skill_data_dir=tmp,
+                        batch_id="batch_no_ff",
+                    ):
+                        pass
+
+                asyncio.run(_run())
+                session = RpaVideoSession(
+                    skill_slug="receive-order",
+                    skill_data_dir=tmp,
+                    batch_id="batch_no_ff",
+                )
+                session._start_ffmpeg_capture()
+                self.assertIn("ffmpeg_not_found", session.warnings)
+        finally:
+            os.environ.pop("OPENCLAW_RECORD_VIDEO", None)
+            config.reset_cache()
+
+    def test_finalize_missing_capture_warning(self) -> None:
         os.environ["OPENCLAW_RECORD_VIDEO"] = "1"
         try:
             with tempfile.TemporaryDirectory() as tmp:
@@ -72,11 +137,11 @@ class TestRpaVideoSession(unittest.TestCase):
                     skill_data_dir=tmp,
                     batch_id="batch_warn",
                 )
-                import asyncio
-
                 asyncio.run(session.finalize())
-                self.assertIn("no_playwright_raw_video", session.warnings)
+                self.assertIn("ffmpeg_capture_missing", session.warnings)
                 self.assertIsNone(session.output_video_path)
+                summ = session.summary()
+                self.assertIsNone(summ["path"])
         finally:
             os.environ.pop("OPENCLAW_RECORD_VIDEO", None)
             config.reset_cache()