From 4a9d358aef47b573287bfcb0c922d7435f72fdd6 Mon Sep 17 00:00:00 2001 From: chendelian <116870791@qq.com> Date: Tue, 2 Jun 2026 14:07:00 +0800 Subject: [PATCH] refactor: ffmpeg desktop capture replaces Playwright recording Co-authored-by: Cursor --- src/jiangchang_skill_core/rpa/browser.py | 7 +- .../rpa/video_session.py | 253 +++++++++++++----- tests/test_video_session.py | 89 +++++- 3 files changed, 271 insertions(+), 78 deletions(-) diff --git a/src/jiangchang_skill_core/rpa/browser.py b/src/jiangchang_skill_core/rpa/browser.py index 86cb8a0..8d4ec77 100644 --- a/src/jiangchang_skill_core/rpa/browser.py +++ b/src/jiangchang_skill_core/rpa/browser.py @@ -1,4 +1,4 @@ -"""统一 persistent context 启动封装。""" +"""统一 persistent context 启动封装（仅浏览器自动化，不负责录屏）。""" from __future__ import annotations @@ -24,12 +24,11 @@ async def launch_persistent_browser( executable_path: str | None = None, headless: bool | None = None, extra_args: list[str] | None = None, - record_video_dir: str | None = None, ) -> "BrowserContext": """用 stealth 参数启动 persistent context + 注入 STEALTH_INIT_SCRIPT。 headless=None 时读 OPENCLAW_BROWSER_HEADLESS。 - record_video_dir 非空则开录屏。 + 录屏由 RpaVideoSession（ffmpeg）负责，本函数不向 Playwright 传递任何录屏参数。 """ if headless is None: v = (os.environ.get("OPENCLAW_BROWSER_HEADLESS") or "0").strip().lower() @@ -52,8 +51,6 @@ async def launch_persistent_browser( ) if ignore is not None: launch_kwargs["ignore_default_args"] = ignore - if record_video_dir: - launch_kwargs["record_video_dir"] = record_video_dir context = await playwright.chromium.launch_persistent_context(**launch_kwargs) if stealth_enabled(): diff --git a/src/jiangchang_skill_core/rpa/video_session.py b/src/jiangchang_skill_core/rpa/video_session.py index bcc4ea2..206d4d7 100644 --- a/src/jiangchang_skill_core/rpa/video_session.py +++ b/src/jiangchang_skill_core/rpa/video_session.py @@ -1,10 +1,14 @@ -"""RPA 运行录屏会话：Playwright raw video → 字幕 + 背景音乐 → 最终 MP4。""" +"""RPA 运行录屏会话：ffmpeg 桌面录制 → 字幕 + 背景音乐 → 最终 MP4。 + +Playwright 仅用于浏览器自动化，不负责录屏。 +""" from __future__ import annotations import asyncio import os import random +import shutil import subprocess import sys import time @@ -15,7 +19,6 @@ from typing import Any, Dict, List, Optional from .. import config -# 与 screencast/composer.py 保持一致的字幕样式 _JIANGCHANG_SUBTITLE_STYLE = ( "FontName=Arial,FontSize=14," "PrimaryColour=&H00FFFFFF,OutlineColour=&H00000000," @@ -27,6 +30,8 @@ _DEFAULT_MUSIC_ROOT = os.environ.get( r"D:\OpenClaw\client-commons\media-assets", ) +_CAPTURE_FRAMERATE = 15 + def _record_video_enabled() -> bool: return config.get_bool("OPENCLAW_RECORD_VIDEO", default=False) @@ -50,7 +55,6 @@ class _StepEntry: def _generate_srt(entries: List[_StepEntry], output_path: str) -> None: - """根据步骤时间线生成 SRT（防重叠）。""" MIN_DURATION = 1.0 GAP = 0.1 steps = sorted(entries, key=lambda e: e.start) @@ -86,16 +90,6 @@ def _escape_srt_path(path: str) -> str: return path.replace("\\", "/").replace(":", "\\:") -def _find_raw_webm(raw_dir: str) -> Optional[Path]: - root = Path(raw_dir) - if not root.is_dir(): - return None - webms = list(root.rglob("*.webm")) - if not webms: - return None - return max(webms, key=lambda p: p.stat().st_size) - - def _run_ffmpeg(cmd: List[str]) -> tuple[bool, str]: try: result = subprocess.run( @@ -113,8 +107,31 @@ def _run_ffmpeg(cmd: List[str]) -> tuple[bool, str]: return True, "" -def _compose_webm_mp4( - webm_path: Path, +def _build_desktop_record_cmd(capture_path: str) -> Optional[List[str]]: + """Windows gdigrab 全桌面录制；非 Windows 返回 None。""" + if sys.platform != "win32": + return None + return [ + "ffmpeg", + "-y", + "-f", + "gdigrab", + "-framerate", + str(_CAPTURE_FRAMERATE), + "-i", + "desktop", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + capture_path, + ] + + +def _compose_capture_mp4( + capture_path: Path, srt_path: Path, output_path: Path, *, @@ -134,39 +151,60 @@ def _compose_webm_mp4( f"[1:a]volume={music_volume},apad[aout]" ) cmd = [ - "ffmpeg", "-y", - "-i", str(webm_path), - "-i", str(music_file), - "-filter_complex", filter_complex, - "-map", "[vout]", - "-map", "[aout]", - "-c:v", "libx264", - "-preset", "fast", - "-crf", "23", - "-c:a", "aac", - "-b:a", "128k", + "ffmpeg", + "-y", + "-i", + str(capture_path), + "-i", + str(music_file), + "-filter_complex", + filter_complex, + "-map", + "[vout]", + "-map", + "[aout]", + "-c:v", + "libx264", + "-preset", + "fast", + "-crf", + "23", + "-c:a", + "aac", + "-b:a", + "128k", "-shortest", - "-pix_fmt", "yuv420p", + "-pix_fmt", + "yuv420p", str(output_path), ] else: cmd = [ - "ffmpeg", "-y", - "-i", str(webm_path), - "-vf", subtitle_filter, - "-c:v", "libx264", - "-preset", "fast", - "-crf", "23", - "-c:a", "aac", - "-b:a", "128k", - "-pix_fmt", "yuv420p", + "ffmpeg", + "-y", + "-i", + str(capture_path), + "-vf", + subtitle_filter, + "-c:v", + "libx264", + "-preset", + "fast", + "-crf", + "23", + "-c:a", + "aac", + "-b:a", + "128k", + "-pix_fmt", + "yuv420p", str(output_path), ] return _run_ffmpeg(cmd) class RpaVideoSession: - """Service/RPA 层统一录屏会话；OPENCLAW_RECORD_VIDEO=1 时启用完整成片流程。""" + """Service/RPA 层统一 ffmpeg 桌面录屏；OPENCLAW_RECORD_VIDEO=1 时启用成片流程。""" def __init__( self, @@ -183,6 +221,8 @@ class RpaVideoSession: self.enabled = _record_video_enabled() self.warnings: List[str] = [] self.output_video_path: Optional[str] = None + self.capture_path: Optional[str] = None + # 已废弃：勿用于 Playwright；始终为 None。 self.record_video_dir: Optional[str] = None self.artifacts: Dict[str, Any] = {} @@ -194,24 +234,27 @@ class RpaVideoSession: self._artifact_root = os.path.join( self.skill_data_dir, "rpa-artifacts", batch_id ) - self._raw_video_dir = os.path.join(self._artifact_root, "raw-video") self._subtitles_dir = os.path.join(self._artifact_root, "subtitles") self._logs_dir = os.path.join(self._artifact_root, "logs") + self._capture_path = os.path.join(self._artifact_root, "capture.mp4") self._srt_path = os.path.join(self._subtitles_dir, f"{self._video_stem}.srt") self._planned_output = os.path.join(self._videos_dir, f"{self._video_stem}.mp4") self._steps: List[_StepEntry] = [] self._t0: Optional[float] = None + self._ffmpeg_proc: Optional[subprocess.Popen] = None if self.enabled: os.makedirs(self._videos_dir, exist_ok=True) - os.makedirs(self._raw_video_dir, exist_ok=True) os.makedirs(self._subtitles_dir, exist_ok=True) os.makedirs(self._logs_dir, exist_ok=True) - self.record_video_dir = self._raw_video_dir + os.makedirs(self._artifact_root, exist_ok=True) def step(self, text: str, *, duration: float = 4.0) -> None: - """记录一步骤字幕（中文说明）。""" + self.add_step(text, duration=duration) + + def add_step(self, text: str, *, duration: float = 4.0) -> None: + """记录一步骤字幕（相对录屏开始时间）。""" if not self.enabled: return if self._t0 is None: @@ -219,71 +262,159 @@ class RpaVideoSession: elapsed = time.monotonic() - self._t0 self._steps.append(_StepEntry(start=elapsed, text=text.strip(), duration=duration)) + def _start_ffmpeg_capture(self) -> None: + if not self.enabled: + return + if not shutil.which("ffmpeg"): + self.warnings.append("ffmpeg_not_found") + return + + cmd = _build_desktop_record_cmd(self._capture_path) + if cmd is None: + self.warnings.append( + "ffmpeg_record_start_failed: desktop capture only supported on Windows" + ) + return + + try: + self._ffmpeg_proc = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + self.capture_path = self._capture_path + except Exception as exc: + self.warnings.append(f"ffmpeg_record_start_failed: {exc}") + self._ffmpeg_proc = None + + def _stop_ffmpeg_capture(self) -> None: + proc = self._ffmpeg_proc + if proc is None: + return + if proc.poll() is None: + try: + if proc.stdin: + proc.stdin.write(b"q") + proc.stdin.flush() + except Exception: + try: + proc.terminate() + except Exception: + pass + try: + proc.wait(timeout=20) + except subprocess.TimeoutExpired: + try: + proc.kill() + except Exception: + pass + try: + proc.wait(timeout=5) + except Exception: + pass + self._ffmpeg_proc = None + await_sleep = 0.3 + time.sleep(await_sleep) + async def __aenter__(self) -> "RpaVideoSession": if self.enabled: self._t0 = time.monotonic() if self.title: - self.step(self.title, duration=3.0) + self.add_step(self.title, duration=3.0) + self._start_ffmpeg_capture() return self async def __aexit__(self, exc_type, exc, tb) -> None: if self.enabled: + self._stop_ffmpeg_capture() await self.finalize() async def finalize(self) -> None: - """在 Playwright context 已 close 后合成最终 MP4；失败只记 warning。""" + """停止录制后合成最终 MP4；失败只记 warning，不抛异常。""" if not self.enabled: + self.artifacts = self.summary() return if not self._steps and self.title: - self.step(self.title) + self.add_step(self.title) try: - await asyncio.sleep(0.5) - webm = _find_raw_webm(self._raw_video_dir) - if webm is None: - self.warnings.append("no_playwright_raw_video") - self.artifacts = self.to_artifact_dict() + capture = Path(self._capture_path) + if not capture.is_file() or capture.stat().st_size <= 0: + self.warnings.append("ffmpeg_capture_missing") + self.artifacts = self.summary() return + self.capture_path = str(capture.resolve()) + entries = list(self._steps) if entries: entries[-1].duration = max(entries[-1].duration, 5.0) - _generate_srt(entries, self._srt_path) + try: + _generate_srt(entries, self._srt_path) + except Exception as exc: + self.warnings.append(f"subtitle_write_failed: {exc}") music = _resolve_music_file() if music is None: self.warnings.append("background_music_missing") - ok, err = _compose_webm_mp4( - webm, - Path(self._srt_path), + srt_file = Path(self._srt_path) + use_srt = srt_file.is_file() and srt_file.stat().st_size > 0 + + ok, err = _compose_capture_mp4( + capture, + srt_file if use_srt else Path(self._srt_path), Path(self._planned_output), music_file=music, ) if not ok and music is not None: self.warnings.append(f"ffmpeg_with_music_failed: {err}") - ok, err = _compose_webm_mp4( - webm, - Path(self._srt_path), + ok, err = _compose_capture_mp4( + capture, + srt_file if use_srt else Path(self._srt_path), Path(self._planned_output), music_file=None, ) + if not ok and not use_srt: + ok, err = _run_ffmpeg( + [ + "ffmpeg", + "-y", + "-i", + str(capture), + "-c:v", + "libx264", + "-preset", + "fast", + "-crf", + "23", + "-pix_fmt", + "yuv420p", + str(self._planned_output), + ] + ) if not ok: self.warnings.append(f"ffmpeg_compose_failed: {err}") - self.artifacts = self.to_artifact_dict() + self.artifacts = self.summary() return self.output_video_path = str(Path(self._planned_output).resolve()) - self.artifacts = self.to_artifact_dict() except Exception as exc: self.warnings.append(f"video_finalize_error: {exc}") - self.artifacts = self.to_artifact_dict() - def to_artifact_dict(self) -> Dict[str, Any]: + self.artifacts = self.summary() + + def summary(self) -> Dict[str, Any]: return { "enabled": self.enabled, "path": self.output_video_path, + "capture_path": self.capture_path, "warnings": list(self.warnings), - "batch_id": self.batch_id, } + + def to_artifact_dict(self) -> Dict[str, Any]: + data = self.summary() + data["batch_id"] = self.batch_id + return data diff --git a/tests/test_video_session.py b/tests/test_video_session.py index 0547ed2..4a7189f 100644 --- a/tests/test_video_session.py +++ b/tests/test_video_session.py @@ -1,10 +1,13 @@ # -*- coding: utf-8 -*- -"""RpaVideoSession 路径与 no-op 行为。""" +"""RpaVideoSession：ffmpeg 桌面录屏，无 Playwright record_video_dir。""" from __future__ import annotations +import asyncio import os +import sys import tempfile import unittest +from unittest.mock import MagicMock, patch from jiangchang_skill_core import config from jiangchang_skill_core.rpa.video_session import RpaVideoSession @@ -27,12 +30,16 @@ class TestRpaVideoSession(unittest.TestCase): self.assertFalse(session.enabled) self.assertIsNone(session.record_video_dir) self.assertIsNone(session.output_video_path) + self.assertIsNone(session.capture_path) + summ = session.summary() + self.assertFalse(summ["enabled"]) + self.assertIsNone(summ["path"]) finally: if old is not None: os.environ["OPENCLAW_RECORD_VIDEO"] = old config.reset_cache() - def test_enabled_paths(self) -> None: + def test_enabled_paths_no_raw_video(self) -> None: os.environ["OPENCLAW_RECORD_VIDEO"] = "1" try: with tempfile.TemporaryDirectory() as tmp: @@ -44,9 +51,10 @@ class TestRpaVideoSession(unittest.TestCase): title="接收订单", ) self.assertTrue(session.enabled) + self.assertIsNone(session.record_video_dir) self.assertTrue( - session.record_video_dir.replace("\\", "/").endswith( - "/rpa-artifacts/exmail_20260602_120000/raw-video" + session._capture_path.replace("\\", "/").endswith( + "/rpa-artifacts/exmail_20260602_120000/capture.mp4" ) ) self.assertTrue( @@ -54,15 +62,72 @@ class TestRpaVideoSession(unittest.TestCase): tmp.replace("\\", "/") + "/videos/" ) ) - self.assertNotIn( - "rpa-artifacts", - session._planned_output.replace("\\", "/").split("/videos/")[-1], - ) finally: os.environ.pop("OPENCLAW_RECORD_VIDEO", None) config.reset_cache() - def test_finalize_missing_webm_only_warning(self) -> None: + @unittest.skipUnless(sys.platform == "win32", "gdigrab desktop capture") + @patch("jiangchang_skill_core.rpa.video_session.shutil.which", return_value="ffmpeg") + @patch("jiangchang_skill_core.rpa.video_session.subprocess.Popen") + def test_enter_starts_ffmpeg_gdigrab(self, mock_popen: MagicMock, _which: MagicMock) -> None: + proc = MagicMock() + proc.poll.return_value = None + proc.stdin = MagicMock() + mock_popen.return_value = proc + + os.environ["OPENCLAW_RECORD_VIDEO"] = "1" + try: + with tempfile.TemporaryDirectory() as tmp: + config.reset_cache() + + async def _run() -> None: + async with RpaVideoSession( + skill_slug="receive-order", + skill_data_dir=tmp, + batch_id="batch_ff", + ) as video: + video.add_step("测试步骤") + with patch.object(video, "finalize", return_value=None): + pass + + asyncio.run(_run()) + self.assertTrue(mock_popen.called) + cmd = mock_popen.call_args[0][0] + self.assertIn("ffmpeg", cmd) + self.assertIn("gdigrab", cmd) + self.assertIn("desktop", cmd) + finally: + os.environ.pop("OPENCLAW_RECORD_VIDEO", None) + config.reset_cache() + + @patch("jiangchang_skill_core.rpa.video_session.shutil.which", return_value=None) + def test_ffmpeg_not_found_warning(self, _which: MagicMock) -> None: + os.environ["OPENCLAW_RECORD_VIDEO"] = "1" + try: + with tempfile.TemporaryDirectory() as tmp: + config.reset_cache() + + async def _run() -> None: + async with RpaVideoSession( + skill_slug="receive-order", + skill_data_dir=tmp, + batch_id="batch_no_ff", + ): + pass + + asyncio.run(_run()) + session = RpaVideoSession( + skill_slug="receive-order", + skill_data_dir=tmp, + batch_id="batch_no_ff", + ) + session._start_ffmpeg_capture() + self.assertIn("ffmpeg_not_found", session.warnings) + finally: + os.environ.pop("OPENCLAW_RECORD_VIDEO", None) + config.reset_cache() + + def test_finalize_missing_capture_warning(self) -> None: os.environ["OPENCLAW_RECORD_VIDEO"] = "1" try: with tempfile.TemporaryDirectory() as tmp: @@ -72,11 +137,11 @@ class TestRpaVideoSession(unittest.TestCase): skill_data_dir=tmp, batch_id="batch_warn", ) - import asyncio - asyncio.run(session.finalize()) - self.assertIn("no_playwright_raw_video", session.warnings) + self.assertIn("ffmpeg_capture_missing", session.warnings) self.assertIsNone(session.output_video_path) + summ = session.summary() + self.assertIsNone(summ["path"]) finally: os.environ.pop("OPENCLAW_RECORD_VIDEO", None) config.reset_cache()