From 857e7e3d73671e1573cc9ae9607c4d8cda538a3c Mon Sep 17 00:00:00 2001 From: chendelian <116870791@qq.com> Date: Mon, 8 Jun 2026 11:50:54 +0800 Subject: [PATCH] feat: improve rpa video timing and narration Co-authored-by: Cursor --- pyproject.toml | 2 +- src/jiangchang_desktop_sdk/__init__.py | 2 +- .../rpa/video_session.py | 56 +++++- tests/test_video_session.py | 162 +++++++++++++++++- 4 files changed, 212 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 231063d..c35ac78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "jiangchang-platform-kit" -version = "1.0.12" +version = "1.0.13" description = "匠厂平台共享组件:Skill 实体 SDK + 桌面应用自动化 SDK" readme = "README.md" requires-python = ">=3.10" diff --git a/src/jiangchang_desktop_sdk/__init__.py b/src/jiangchang_desktop_sdk/__init__.py index 87c2562..a0a4311 100644 --- a/src/jiangchang_desktop_sdk/__init__.py +++ b/src/jiangchang_desktop_sdk/__init__.py @@ -23,7 +23,7 @@ __all__ = [ "LaunchError", "GatewayDownError", ] -__version__ = "1.0.12" +__version__ = "1.0.13" def __getattr__(name: str): diff --git a/src/jiangchang_skill_core/rpa/video_session.py b/src/jiangchang_skill_core/rpa/video_session.py index 12ec5c9..5cb44d9 100644 --- a/src/jiangchang_skill_core/rpa/video_session.py +++ b/src/jiangchang_skill_core/rpa/video_session.py @@ -45,6 +45,10 @@ _VOICEOVER_MAX_CHARS = 60 _VOICEOVER_DEDUP_SECONDS = 10.0 _VOICEOVER_MIN_GAP_SECONDS = 2.0 +_INTRO_BUFFER_SECONDS = 5.0 +_INTRO_STABILIZE_SECONDS = 1.0 +_OUTRO_BUFFER_SECONDS = 5.0 + _VOICEOVER_SKIP_SUBSTRINGS = ( "等待 1-5s", "跳过重复", @@ -54,6 +58,25 @@ _VOICEOVER_SKIP_SUBSTRINGS = ( "失败", ) +_VOICEOVER_IMPORTANT_SUBSTRINGS = ( + "开始", + "打开", + "启动", + "定位", + "输入", + "点击", + "搜索", + "等待搜索结果", + "检查登录状态", + "处理验证码", + "等待人工验证", + "解析", + "提取", + "采集完成", + "部分完成", + "完成", +) + # 明显技术日志：时间戳前缀、纯数字进度等 _TECH_LOG_RE = re.compile( r"^\d{4}-\d{2}-\d{2}|^\[\w+\]|^DEBUG:|^INFO:|^WARN:|^ERROR:" @@ -173,10 +196,17 @@ def _voiceover_text_for_step(text: str) -> Optional[str]: return cleaned +def _is_important_voiceover(text: str) -> bool: + for important in _VOICEOVER_IMPORTANT_SUBSTRINGS: + if important in text: + return True + return False + + def _select_voiceover_clips( entries: List[_StepEntry], ) -> List[tuple[_StepEntry, str]]: - """按时间排序，去重并保证旁白间隔。""" + """按时间排序，去重并保证旁白间隔；关键动作可绕过最小间隔。""" result: List[tuple[_StepEntry, str]] = [] last_text_at: Dict[str, float] = {} last_spoken_at = -_VOICEOVER_MIN_GAP_SECONDS @@ -188,7 +218,10 @@ def _select_voiceover_clips( prev = last_text_at.get(spoken) if prev is not None and entry.start - prev < _VOICEOVER_DEDUP_SECONDS: continue - if entry.start - last_spoken_at < _VOICEOVER_MIN_GAP_SECONDS: + if ( + not _is_important_voiceover(spoken) + and entry.start - last_spoken_at < _VOICEOVER_MIN_GAP_SECONDS + ): continue last_text_at[spoken] = entry.start last_spoken_at = entry.start @@ -534,11 +567,13 @@ class RpaVideoSession: skill_data_dir: str, batch_id: str, title: str = "", + closing_title: str = "", ) -> None: self.skill_slug = skill_slug self.skill_data_dir = os.path.abspath(skill_data_dir) self.batch_id = batch_id self.title = title + self.closing_title = closing_title self.enabled = _record_video_enabled() self.warnings: List[str] = [] self.output_video_path: Optional[str] = None @@ -684,16 +719,29 @@ class RpaVideoSession: self._close_ffmpeg_record_log() time.sleep(0.3) + async def _safe_add_step(self, text: str, *, duration: float = 4.0) -> None: + try: + self.add_step(text, duration=duration) + except Exception: + pass + async def __aenter__(self) -> "RpaVideoSession": if self.enabled: self._t0 = time.monotonic() - if self.title: - self.add_step(self.title, duration=3.0) self._start_ffmpeg_capture() + await asyncio.sleep(_INTRO_STABILIZE_SECONDS) + if self.title: + await self._safe_add_step(self.title, duration=4.0) + intro_wait = _INTRO_BUFFER_SECONDS - _INTRO_STABILIZE_SECONDS + if intro_wait > 0: + await asyncio.sleep(intro_wait) return self async def __aexit__(self, exc_type, exc, tb) -> None: if self.enabled: + if self.closing_title: + await self._safe_add_step(self.closing_title, duration=4.0) + await asyncio.sleep(_OUTRO_BUFFER_SECONDS) self._stop_ffmpeg_capture() await self.finalize() diff --git a/tests/test_video_session.py b/tests/test_video_session.py index 086fd9a..3783957 100644 --- a/tests/test_video_session.py +++ b/tests/test_video_session.py @@ -8,13 +8,16 @@ import sys import tempfile import unittest from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch from jiangchang_skill_core import config from jiangchang_skill_core.rpa import video_session as vs from jiangchang_skill_core.rpa.video_session import ( RpaVideoSession, _StepEntry, + _INTRO_BUFFER_SECONDS, + _INTRO_STABILIZE_SECONDS, + _OUTRO_BUFFER_SECONDS, _compose_capture_mp4, _music_fade_params, _select_voiceover_clips, @@ -225,15 +228,38 @@ class TestVoiceoverTextFilter(unittest.TestCase): def test_min_gap_two_seconds(self) -> None: entries = [ - _StepEntry(0.0, "开始采集"), - _StepEntry(1.0, "启动浏览器"), + _StepEntry(0.0, "获取1688账号"), + _StepEntry(1.0, "连接CDP"), _StepEntry(3.0, "检查登录状态"), ] clips = _select_voiceover_clips(entries) self.assertEqual(len(clips), 2) - self.assertEqual(clips[0][1], "开始采集") + self.assertEqual(clips[0][1], "获取1688账号") self.assertEqual(clips[1][1], "检查登录状态") + def test_important_bypasses_min_gap(self) -> None: + entries = [ + _StepEntry(0.0, "定位搜索框"), + _StepEntry(0.5, "输入关键词：宁德电池"), + _StepEntry(1.0, "点击搜索"), + _StepEntry(1.5, "等待搜索结果"), + ] + clips = _select_voiceover_clips(entries) + texts = [t for _, t in clips] + self.assertEqual(len(clips), 4) + self.assertIn("输入关键词：宁德电池", texts) + self.assertIn("点击搜索", texts) + + def test_noise_still_skipped_with_important_nearby(self) -> None: + entries = [ + _StepEntry(0.0, "点击搜索"), + _StepEntry(0.5, "跳过重复店铺"), + _StepEntry(1.0, "写入联系人库"), + ] + clips = _select_voiceover_clips(entries) + texts = [t for _, t in clips] + self.assertEqual(texts, ["点击搜索"]) + class TestComposeCaptureMp4(unittest.TestCase): def setUp(self) -> None: @@ -343,6 +369,134 @@ class TestComposeCaptureMp4(unittest.TestCase): self.assertNotIn("-stream_loop", cmd) +class TestVideoSessionLifecycle(unittest.TestCase): + def setUp(self) -> None: + config.reset_cache() + + def tearDown(self) -> None: + os.environ.pop("OPENCLAW_RECORD_VIDEO", None) + config.reset_cache() + + @patch("jiangchang_skill_core.rpa.video_session.asyncio.sleep", new_callable=AsyncMock) + @patch.object(RpaVideoSession, "_start_ffmpeg_capture") + @patch.object(RpaVideoSession, "_stop_ffmpeg_capture") + @patch.object(RpaVideoSession, "finalize", new_callable=AsyncMock) + def test_enter_starts_capture_before_title( + self, + mock_finalize: AsyncMock, + mock_stop: MagicMock, + mock_start: MagicMock, + mock_sleep: AsyncMock, + ) -> None: + os.environ["OPENCLAW_RECORD_VIDEO"] = "1" + call_order: list[str] = [] + mock_start.side_effect = lambda: call_order.append("start_capture") + + with tempfile.TemporaryDirectory() as tmp: + config.reset_cache() + session = RpaVideoSession( + skill_slug="test", + skill_data_dir=tmp, + batch_id="batch_intro", + title="开始执行测试", + ) + original_add = session.add_step + + def _track_add(text: str, *, duration: float = 4.0) -> None: + call_order.append(f"add_step:{text}") + original_add(text, duration=duration) + + with patch.object(session, "add_step", side_effect=_track_add): + asyncio.run(session.__aenter__()) + + self.assertEqual(call_order[0], "start_capture") + self.assertIn("add_step:开始执行测试", call_order) + mock_sleep.assert_awaited() + total_sleep = sum(c.args[0] for c in mock_sleep.await_args_list if c.args) + self.assertAlmostEqual(total_sleep, _INTRO_BUFFER_SECONDS) + + @patch("jiangchang_skill_core.rpa.video_session.asyncio.sleep", new_callable=AsyncMock) + @patch.object(RpaVideoSession, "_start_ffmpeg_capture") + def test_enter_no_title_still_buffers( + self, + mock_start: MagicMock, + mock_sleep: AsyncMock, + ) -> None: + os.environ["OPENCLAW_RECORD_VIDEO"] = "1" + with tempfile.TemporaryDirectory() as tmp: + config.reset_cache() + session = RpaVideoSession( + skill_slug="test", + skill_data_dir=tmp, + batch_id="batch_no_title", + ) + with patch.object(session, "add_step") as mock_add: + asyncio.run(session.__aenter__()) + mock_start.assert_called_once() + mock_add.assert_not_called() + total_sleep = sum(c.args[0] for c in mock_sleep.await_args_list if c.args) + self.assertAlmostEqual(total_sleep, _INTRO_BUFFER_SECONDS) + + @patch("jiangchang_skill_core.rpa.video_session.asyncio.sleep", new_callable=AsyncMock) + @patch.object(RpaVideoSession, "_stop_ffmpeg_capture") + @patch.object(RpaVideoSession, "finalize", new_callable=AsyncMock) + def test_exit_closing_title_before_stop( + self, + mock_finalize: AsyncMock, + mock_stop: MagicMock, + mock_sleep: AsyncMock, + ) -> None: + os.environ["OPENCLAW_RECORD_VIDEO"] = "1" + call_order: list[str] = [] + mock_stop.side_effect = lambda: call_order.append("stop_capture") + + with tempfile.TemporaryDirectory() as tmp: + config.reset_cache() + session = RpaVideoSession( + skill_slug="test", + skill_data_dir=tmp, + batch_id="batch_outro", + closing_title="任务已完成", + ) + original_add = session.add_step + + def _track_add(text: str, *, duration: float = 4.0) -> None: + call_order.append(f"add_step:{text}") + original_add(text, duration=duration) + + with patch.object(session, "add_step", side_effect=_track_add): + asyncio.run(session.__aexit__(None, None, None)) + + self.assertIn("add_step:任务已完成", call_order) + self.assertEqual(call_order[-1], "stop_capture") + total_sleep = sum(c.args[0] for c in mock_sleep.await_args_list if c.args) + self.assertAlmostEqual(total_sleep, _OUTRO_BUFFER_SECONDS) + + @patch("jiangchang_skill_core.rpa.video_session.asyncio.sleep", new_callable=AsyncMock) + @patch.object(RpaVideoSession, "_stop_ffmpeg_capture") + @patch.object(RpaVideoSession, "finalize", new_callable=AsyncMock) + def test_exit_no_closing_title_still_outro_buffer( + self, + mock_finalize: AsyncMock, + mock_stop: MagicMock, + mock_sleep: AsyncMock, + ) -> None: + os.environ["OPENCLAW_RECORD_VIDEO"] = "1" + with tempfile.TemporaryDirectory() as tmp: + config.reset_cache() + session = RpaVideoSession( + skill_slug="test", + skill_data_dir=tmp, + batch_id="batch_outro_empty", + ) + with patch.object(session, "add_step") as mock_add: + asyncio.run(session.__aexit__(None, None, None)) + mock_add.assert_not_called() + mock_stop.assert_called_once() + total_sleep = sum(c.args[0] for c in mock_sleep.await_args_list if c.args) + self.assertAlmostEqual(total_sleep, _OUTRO_BUFFER_SECONDS) + + class TestFinalizeDegradation(unittest.TestCase): def setUp(self) -> None: config.reset_cache()