diff --git a/CHANGELOG.md b/CHANGELOG.md index 7970adc..c94546c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ 本文件对应技能市场 **「更新日志」** Tab。 +## 1.1.19 + +- **修复 1.1.18 引入的采集回归**：把 `_persist_contracts` 里的入队调用移到独立事务 `_enqueue_download_tasks_safe`，FK 失败不再回滚 `contracts` / `materials_content` 落库；返回 JSON 新增 `download_queue` 字段方便 UI 直接看到入队结果 +- 新增 4 个测试覆盖 `_enqueue_download_tasks_safe`（basic / idempotent / fk_failure / empty_list），合计 54/54 + ## 1.1.18 - **一键式"运维 + 材料管理"全打通** diff --git a/SKILL.md b/SKILL.md index 80e32ce..6a6f574 100644 --- a/SKILL.md +++ b/SKILL.md @@ -1,7 +1,7 @@ --- name: 合同到期智能预警与续聘辅助 description: "当用户说要扫描即将到期的合同、查 90/60/30 天预警名单、给某位教师生成续聘任务书、下发单位版续聘清单、从 SmartCampus HR 平台抓合同数据、解析续聘材料 PDF/Word、查看本月合同到期人员清单时，加载本技能。支持定时扫描（每日凌晨）+ 数据管理 + Agent 三入口，覆盖合同到期智能预警与续聘辅助全流程。" -version: 1.1.18 +version: 1.1.19 author: 深圳匠厂科技有限公司 metadata: openclaw: diff --git a/scripts/service/platform_collector.py b/scripts/service/platform_collector.py index cb1220c..28354af 100644 --- a/scripts/service/platform_collector.py +++ b/scripts/service/platform_collector.py @@ -763,11 +763,21 @@ def collect_all( ctx.close() saved = _persist_contracts(contracts) + logger.info( + "platform_collector: contracts 落库完成 saved=%d / total_fetched=%d", + saved, len(contracts), + ) + + # v1.1.18：每个合同入队 download_tasks（独立事务，失败不影响主流程） + contract_ids = [c["contract_id"] for c in contracts if c.get("contract_id")] + enqueue_summary = _enqueue_download_tasks_safe(contract_ids) + return { "platform_url": PLATFORM_URL, "fetched_rows": fetched_rows, "saved_contracts": saved, "downloaded_materials": downloaded, + "download_queue": enqueue_summary, "contracts": contracts, } @@ -865,14 +875,78 @@ def _persist_contracts(contracts: list[dict]) -> int: datetime.now().isoformat(timespec="seconds"), ), ) - # v1.1.18：每个合同入队 download_tasks（即使已抓到卡片也入队， - # process_queue 失败时可用作重试锚点；enqueue_download_task 幂等） - repo.enqueue_download_task(conn, c["contract_id"], "platform_button") + # v1.1.18 入队 download_tasks 已移到 collect_all 末尾的独立事务， + # 见 _enqueue_download_tasks_safe() return len(contracts) finally: conn.close() +def _enqueue_download_tasks_safe( + contract_ids: list[str], conn: sqlite3.Connection | None = None, +) -> dict: + """v1.1.18：每个合同入队 download_tasks。 + + 在独立事务里完成，不影响 _persist_contracts 主流程（contracts / materials_content）。 + 任一入队失败 → 记 warning + 计入 failed，但不影响整体返回。 + + conn: 可选传入连接（用于测试 in-memory DB）。None → 走 get_connection()。 + 若传入 conn，调用方负责生命周期（不会 close）。 + + Returns: + {enqueued: N, skipped: M, failed: K, errors: [...]} + """ + if not contract_ids: + return {"enqueued": 0, "skipped": 0, "failed": 0, "errors": []} + + from db.connection import get_connection + + enqueued = 0 + skipped = 0 + failed = 0 + errors: list[str] = [] + + owns_conn = conn is None + if owns_conn: + conn = get_connection() + try: + for cid in contract_ids: + try: + # 入队前查一下是否已存在 pending/running → 决定计入 skipped 还是 enqueued + pre_existing = conn.execute( + """SELECT 1 FROM download_tasks + WHERE contract_id = ? AND kind = ? + AND status IN ('pending', 'running') LIMIT 1""", + (cid, "platform_button"), + ).fetchone() + repo.enqueue_download_task(conn, cid, "platform_button") + if pre_existing: + skipped += 1 + else: + enqueued += 1 + except Exception as e: + failed += 1 + err = f"{cid}: {type(e).__name__}: {e}" + errors.append(err) + logger.warning( + "platform_collector.enqueue: 入队 %s 失败: %s", cid, e, + ) + if owns_conn: + conn.commit() + finally: + if owns_conn: + conn.close() + + logger.info( + "platform_collector.enqueue: 入队结果 enqueued=%d skipped=%d failed=%d (total=%d)", + enqueued, skipped, failed, len(contract_ids), + ) + return { + "enqueued": enqueued, "skipped": skipped, + "failed": failed, "errors": errors[:10], + } + + def _download_materials_for_row( page, row_idx: int, contract_id: str, mat_btn_count: int, ) -> list[dict]: diff --git a/scripts/util/constants.py b/scripts/util/constants.py index d24ed88..b2d5578 100644 --- a/scripts/util/constants.py +++ b/scripts/util/constants.py @@ -6,7 +6,7 @@ """ SKILL_SLUG = "alert-contract-expiring-hr" -SKILL_VERSION = "1.1.18" +SKILL_VERSION = "1.1.19" LOG_LOGGER_NAME = "openclaw.skill.alert_contract_expiring_hr" PLATFORM_KIT_MIN_VERSION = "1.2.11" diff --git a/tests/run_tests.py b/tests/run_tests.py index b4cfa58..d1872bb 100644 --- a/tests/run_tests.py +++ b/tests/run_tests.py @@ -1192,6 +1192,85 @@ def test_download_tasks_count_by_status() -> tuple[bool, str]: conn.close() +def test_enqueue_download_tasks_safe_basic() -> tuple[bool, str]: + """_enqueue_download_tasks_safe: 全部成功入队。""" + from service.platform_collector import _enqueue_download_tasks_safe + + # 准备 3 个合同（in-memory DB，函数不会关闭） + conn = _make_conn() + for i in range(3): + conn.execute( + f"""INSERT INTO contracts + (contract_id, employee_id, teacher_name, unit_id, unit_name, + position, contract_type, contract_start, contract_end, + trial_period_end, status) + VALUES ('ENQ-{i}', 'T{i}', 'X', 'U{i}', 'Y', '', '', + '2024-01-01', '2026-04-15', '2024-04-15', '可续聘')""" + ) + conn.commit() + + summary = _enqueue_download_tasks_safe(["ENQ-0", "ENQ-1", "ENQ-2"], conn=conn) + if summary["enqueued"] != 3: + return False, f"enqueued={summary['enqueued']} (want 3) - {summary}" + if summary["failed"] != 0: + return False, f"failed={summary['failed']} - {summary}" + if summary["errors"]: + return False, f"errors={summary['errors']}" + return True, f"3 个合同全部入队: {summary}" + + +def test_enqueue_download_tasks_safe_idempotent() -> tuple[bool, str]: + """_enqueue_download_tasks_safe: 同 contract_id 第二次调用 → skipped。""" + from service.platform_collector import _enqueue_download_tasks_safe + + conn = _make_conn() + conn.execute( + f"""INSERT INTO contracts + (contract_id, employee_id, teacher_name, unit_id, unit_name, + position, contract_type, contract_start, contract_end, + trial_period_end, status) + VALUES ('ENQ-IDM', 'T1', 'X', 'U1', 'Y', '', '', + '2024-01-01', '2026-04-15', '2024-04-15', '可续聘')""" + ) + conn.commit() + + s1 = _enqueue_download_tasks_safe(["ENQ-IDM"], conn=conn) + s2 = _enqueue_download_tasks_safe(["ENQ-IDM"], conn=conn) + if s1["enqueued"] != 1: + return False, f"第一次 enqueued={s1['enqueued']} (want 1) - {s1}" + if s2["skipped"] != 1: + return False, f"第二次 skipped={s2['skipped']} (want 1) - {s2}" + if s2["enqueued"] != 0: + return False, f"第二次不应新增 enqueued={s2['enqueued']} - {s2}" + return True, f"幂等 OK: 第一次 {s1}, 第二次 {s2}" + + +def test_enqueue_download_tasks_safe_fk_failure() -> tuple[bool, str]: + """_enqueue_download_tasks_safe: contract_id 不存在 → 计入 failed，不抛异常。""" + from service.platform_collector import _enqueue_download_tasks_safe + + # 不插入合同，直接入队 → FK 失败（in-memory DB，函数不关闭） + conn = _make_conn() + summary = _enqueue_download_tasks_safe(["DOES-NOT-EXIST"], conn=conn) + if summary["failed"] != 1: + return False, f"failed={summary['failed']} (want 1) - {summary}" + if not summary["errors"]: + return False, f"errors 应非空: {summary}" + if "FOREIGN KEY" not in summary["errors"][0] and "foreign key" not in summary["errors"][0].lower(): + return False, f"错误信息应提及 FOREIGN KEY: {summary['errors'][0]}" + return True, f"FK 失败被吞掉: failed=1, errors[0]={summary['errors'][0][:80]}" + + +def test_enqueue_download_tasks_safe_empty_list() -> tuple[bool, str]: + """_enqueue_download_tasks_safe: 空列表 → no-op，不开 DB 连接。""" + from service.platform_collector import _enqueue_download_tasks_safe + + summary = _enqueue_download_tasks_safe([]) + if summary != {"enqueued": 0, "skipped": 0, "failed": 0, "errors": []}: + return False, f"空列表返回值异常: {summary}" + return True, "空列表 no-op OK" + + # ---------- v1.1.18: show-config / logs-list ---------- @@ -1457,6 +1536,10 @@ def main() -> int: _run("download_tasks/claim_atomic", test_download_tasks_claim_atomic) _run("download_tasks/mark_success_and_failed", test_download_tasks_mark_success_and_failed) _run("download_tasks/count_by_status", test_download_tasks_count_by_status) + _run("download_tasks/enqueue_safe_basic", test_enqueue_download_tasks_safe_basic) + _run("download_tasks/enqueue_safe_idempotent", test_enqueue_download_tasks_safe_idempotent) + _run("download_tasks/enqueue_safe_fk_failure", test_enqueue_download_tasks_safe_fk_failure) + _run("download_tasks/enqueue_safe_empty_list", test_enqueue_download_tasks_safe_empty_list) _run("show_config/masks_tokens", test_show_config_masks_tokens) _run("show_config/skips_empty", test_show_config_skips_empty) _run("show_config/skips_comments", test_show_config_skips_comments)