feat: improve rpa video timing and narration

Co-authored-by: Cursor <cursoragent@cursor.com>

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "jiangchang-platform-kit"
-version = "1.0.12"
+version = "1.0.13"
 description = "匠厂平台共享组件:Skill 实体 SDK + 桌面应用自动化 SDK"
 readme = "README.md"
 requires-python = ">=3.10"

src/jiangchang_desktop_sdk/__init__.py

@@ -23,7 +23,7 @@ __all__ = [
     "LaunchError",
     "GatewayDownError",
 ]
-__version__ = "1.0.12"
+__version__ = "1.0.13"
 
 
 def __getattr__(name: str):

src/jiangchang_skill_core/rpa/video_session.py

@@ -45,6 +45,10 @@ _VOICEOVER_MAX_CHARS = 60
 _VOICEOVER_DEDUP_SECONDS = 10.0
 _VOICEOVER_MIN_GAP_SECONDS = 2.0
 
+_INTRO_BUFFER_SECONDS = 5.0
+_INTRO_STABILIZE_SECONDS = 1.0
+_OUTRO_BUFFER_SECONDS = 5.0
+
 _VOICEOVER_SKIP_SUBSTRINGS = (
     "等待 1-5s",
     "跳过重复",
@@ -54,6 +58,25 @@ _VOICEOVER_SKIP_SUBSTRINGS = (
     "失败",
 )
 
+_VOICEOVER_IMPORTANT_SUBSTRINGS = (
+    "开始",
+    "打开",
+    "启动",
+    "定位",
+    "输入",
+    "点击",
+    "搜索",
+    "等待搜索结果",
+    "检查登录状态",
+    "处理验证码",
+    "等待人工验证",
+    "解析",
+    "提取",
+    "采集完成",
+    "部分完成",
+    "完成",
+)
+
 # 明显技术日志：时间戳前缀、纯数字进度等
 _TECH_LOG_RE = re.compile(
     r"^\d{4}-\d{2}-\d{2}|^\[\w+\]|^DEBUG:|^INFO:|^WARN:|^ERROR:"
@@ -173,10 +196,17 @@ def _voiceover_text_for_step(text: str) -> Optional[str]:
     return cleaned
 
 
+def _is_important_voiceover(text: str) -> bool:
+    for important in _VOICEOVER_IMPORTANT_SUBSTRINGS:
+        if important in text:
+            return True
+    return False
+
+
 def _select_voiceover_clips(
     entries: List[_StepEntry],
 ) -> List[tuple[_StepEntry, str]]:
-    """按时间排序，去重并保证旁白间隔。"""
+    """按时间排序，去重并保证旁白间隔；关键动作可绕过最小间隔。"""
     result: List[tuple[_StepEntry, str]] = []
     last_text_at: Dict[str, float] = {}
     last_spoken_at = -_VOICEOVER_MIN_GAP_SECONDS
@@ -188,7 +218,10 @@ def _select_voiceover_clips(
         prev = last_text_at.get(spoken)
         if prev is not None and entry.start - prev < _VOICEOVER_DEDUP_SECONDS:
             continue
-        if entry.start - last_spoken_at < _VOICEOVER_MIN_GAP_SECONDS:
+        if (
+            not _is_important_voiceover(spoken)
+            and entry.start - last_spoken_at < _VOICEOVER_MIN_GAP_SECONDS
+        ):
             continue
         last_text_at[spoken] = entry.start
         last_spoken_at = entry.start
@@ -534,11 +567,13 @@ class RpaVideoSession:
         skill_data_dir: str,
         batch_id: str,
         title: str = "",
+        closing_title: str = "",
     ) -> None:
         self.skill_slug = skill_slug
         self.skill_data_dir = os.path.abspath(skill_data_dir)
         self.batch_id = batch_id
         self.title = title
+        self.closing_title = closing_title
         self.enabled = _record_video_enabled()
         self.warnings: List[str] = []
         self.output_video_path: Optional[str] = None
@@ -684,16 +719,29 @@ class RpaVideoSession:
         self._close_ffmpeg_record_log()
         time.sleep(0.3)
 
+    async def _safe_add_step(self, text: str, *, duration: float = 4.0) -> None:
+        try:
+            self.add_step(text, duration=duration)
+        except Exception:
+            pass
+
     async def __aenter__(self) -> "RpaVideoSession":
         if self.enabled:
             self._t0 = time.monotonic()
-            if self.title:
-                self.add_step(self.title, duration=3.0)
             self._start_ffmpeg_capture()
+            await asyncio.sleep(_INTRO_STABILIZE_SECONDS)
+            if self.title:
+                await self._safe_add_step(self.title, duration=4.0)
+            intro_wait = _INTRO_BUFFER_SECONDS - _INTRO_STABILIZE_SECONDS
+            if intro_wait > 0:
+                await asyncio.sleep(intro_wait)
         return self
 
     async def __aexit__(self, exc_type, exc, tb) -> None:
         if self.enabled:
+            if self.closing_title:
+                await self._safe_add_step(self.closing_title, duration=4.0)
+            await asyncio.sleep(_OUTRO_BUFFER_SECONDS)
             self._stop_ffmpeg_capture()
             await self.finalize()
 

tests/test_video_session.py

@@ -8,13 +8,16 @@ import sys
 import tempfile
 import unittest
 from pathlib import Path
-from unittest.mock import MagicMock, patch
+from unittest.mock import AsyncMock, MagicMock, patch
 
 from jiangchang_skill_core import config
 from jiangchang_skill_core.rpa import video_session as vs
 from jiangchang_skill_core.rpa.video_session import (
     RpaVideoSession,
     _StepEntry,
+    _INTRO_BUFFER_SECONDS,
+    _INTRO_STABILIZE_SECONDS,
+    _OUTRO_BUFFER_SECONDS,
     _compose_capture_mp4,
     _music_fade_params,
     _select_voiceover_clips,
@@ -225,15 +228,38 @@ class TestVoiceoverTextFilter(unittest.TestCase):
 
     def test_min_gap_two_seconds(self) -> None:
         entries = [
-            _StepEntry(0.0, "开始采集"),
-            _StepEntry(1.0, "启动浏览器"),
+            _StepEntry(0.0, "获取1688账号"),
+            _StepEntry(1.0, "连接CDP"),
             _StepEntry(3.0, "检查登录状态"),
         ]
         clips = _select_voiceover_clips(entries)
         self.assertEqual(len(clips), 2)
-        self.assertEqual(clips[0][1], "开始采集")
+        self.assertEqual(clips[0][1], "获取1688账号")
         self.assertEqual(clips[1][1], "检查登录状态")
 
+    def test_important_bypasses_min_gap(self) -> None:
+        entries = [
+            _StepEntry(0.0, "定位搜索框"),
+            _StepEntry(0.5, "输入关键词：宁德电池"),
+            _StepEntry(1.0, "点击搜索"),
+            _StepEntry(1.5, "等待搜索结果"),
+        ]
+        clips = _select_voiceover_clips(entries)
+        texts = [t for _, t in clips]
+        self.assertEqual(len(clips), 4)
+        self.assertIn("输入关键词：宁德电池", texts)
+        self.assertIn("点击搜索", texts)
+
+    def test_noise_still_skipped_with_important_nearby(self) -> None:
+        entries = [
+            _StepEntry(0.0, "点击搜索"),
+            _StepEntry(0.5, "跳过重复店铺"),
+            _StepEntry(1.0, "写入联系人库"),
+        ]
+        clips = _select_voiceover_clips(entries)
+        texts = [t for _, t in clips]
+        self.assertEqual(texts, ["点击搜索"])
+
 
 class TestComposeCaptureMp4(unittest.TestCase):
     def setUp(self) -> None:
@@ -343,6 +369,134 @@ class TestComposeCaptureMp4(unittest.TestCase):
         self.assertNotIn("-stream_loop", cmd)
 
 
+class TestVideoSessionLifecycle(unittest.TestCase):
+    def setUp(self) -> None:
+        config.reset_cache()
+
+    def tearDown(self) -> None:
+        os.environ.pop("OPENCLAW_RECORD_VIDEO", None)
+        config.reset_cache()
+
+    @patch("jiangchang_skill_core.rpa.video_session.asyncio.sleep", new_callable=AsyncMock)
+    @patch.object(RpaVideoSession, "_start_ffmpeg_capture")
+    @patch.object(RpaVideoSession, "_stop_ffmpeg_capture")
+    @patch.object(RpaVideoSession, "finalize", new_callable=AsyncMock)
+    def test_enter_starts_capture_before_title(
+        self,
+        mock_finalize: AsyncMock,
+        mock_stop: MagicMock,
+        mock_start: MagicMock,
+        mock_sleep: AsyncMock,
+    ) -> None:
+        os.environ["OPENCLAW_RECORD_VIDEO"] = "1"
+        call_order: list[str] = []
+        mock_start.side_effect = lambda: call_order.append("start_capture")
+
+        with tempfile.TemporaryDirectory() as tmp:
+            config.reset_cache()
+            session = RpaVideoSession(
+                skill_slug="test",
+                skill_data_dir=tmp,
+                batch_id="batch_intro",
+                title="开始执行测试",
+            )
+            original_add = session.add_step
+
+            def _track_add(text: str, *, duration: float = 4.0) -> None:
+                call_order.append(f"add_step:{text}")
+                original_add(text, duration=duration)
+
+            with patch.object(session, "add_step", side_effect=_track_add):
+                asyncio.run(session.__aenter__())
+
+        self.assertEqual(call_order[0], "start_capture")
+        self.assertIn("add_step:开始执行测试", call_order)
+        mock_sleep.assert_awaited()
+        total_sleep = sum(c.args[0] for c in mock_sleep.await_args_list if c.args)
+        self.assertAlmostEqual(total_sleep, _INTRO_BUFFER_SECONDS)
+
+    @patch("jiangchang_skill_core.rpa.video_session.asyncio.sleep", new_callable=AsyncMock)
+    @patch.object(RpaVideoSession, "_start_ffmpeg_capture")
+    def test_enter_no_title_still_buffers(
+        self,
+        mock_start: MagicMock,
+        mock_sleep: AsyncMock,
+    ) -> None:
+        os.environ["OPENCLAW_RECORD_VIDEO"] = "1"
+        with tempfile.TemporaryDirectory() as tmp:
+            config.reset_cache()
+            session = RpaVideoSession(
+                skill_slug="test",
+                skill_data_dir=tmp,
+                batch_id="batch_no_title",
+            )
+            with patch.object(session, "add_step") as mock_add:
+                asyncio.run(session.__aenter__())
+            mock_start.assert_called_once()
+            mock_add.assert_not_called()
+            total_sleep = sum(c.args[0] for c in mock_sleep.await_args_list if c.args)
+            self.assertAlmostEqual(total_sleep, _INTRO_BUFFER_SECONDS)
+
+    @patch("jiangchang_skill_core.rpa.video_session.asyncio.sleep", new_callable=AsyncMock)
+    @patch.object(RpaVideoSession, "_stop_ffmpeg_capture")
+    @patch.object(RpaVideoSession, "finalize", new_callable=AsyncMock)
+    def test_exit_closing_title_before_stop(
+        self,
+        mock_finalize: AsyncMock,
+        mock_stop: MagicMock,
+        mock_sleep: AsyncMock,
+    ) -> None:
+        os.environ["OPENCLAW_RECORD_VIDEO"] = "1"
+        call_order: list[str] = []
+        mock_stop.side_effect = lambda: call_order.append("stop_capture")
+
+        with tempfile.TemporaryDirectory() as tmp:
+            config.reset_cache()
+            session = RpaVideoSession(
+                skill_slug="test",
+                skill_data_dir=tmp,
+                batch_id="batch_outro",
+                closing_title="任务已完成",
+            )
+            original_add = session.add_step
+
+            def _track_add(text: str, *, duration: float = 4.0) -> None:
+                call_order.append(f"add_step:{text}")
+                original_add(text, duration=duration)
+
+            with patch.object(session, "add_step", side_effect=_track_add):
+                asyncio.run(session.__aexit__(None, None, None))
+
+        self.assertIn("add_step:任务已完成", call_order)
+        self.assertEqual(call_order[-1], "stop_capture")
+        total_sleep = sum(c.args[0] for c in mock_sleep.await_args_list if c.args)
+        self.assertAlmostEqual(total_sleep, _OUTRO_BUFFER_SECONDS)
+
+    @patch("jiangchang_skill_core.rpa.video_session.asyncio.sleep", new_callable=AsyncMock)
+    @patch.object(RpaVideoSession, "_stop_ffmpeg_capture")
+    @patch.object(RpaVideoSession, "finalize", new_callable=AsyncMock)
+    def test_exit_no_closing_title_still_outro_buffer(
+        self,
+        mock_finalize: AsyncMock,
+        mock_stop: MagicMock,
+        mock_sleep: AsyncMock,
+    ) -> None:
+        os.environ["OPENCLAW_RECORD_VIDEO"] = "1"
+        with tempfile.TemporaryDirectory() as tmp:
+            config.reset_cache()
+            session = RpaVideoSession(
+                skill_slug="test",
+                skill_data_dir=tmp,
+                batch_id="batch_outro_empty",
+            )
+            with patch.object(session, "add_step") as mock_add:
+                asyncio.run(session.__aexit__(None, None, None))
+            mock_add.assert_not_called()
+            mock_stop.assert_called_once()
+            total_sleep = sum(c.args[0] for c in mock_sleep.await_args_list if c.args)
+            self.assertAlmostEqual(total_sleep, _OUTRO_BUFFER_SECONDS)
+
+
 class TestFinalizeDegradation(unittest.TestCase):
     def setUp(self) -> None:
         config.reset_cache()