"""Playwright RPA:SmartCampus HR 平台下载队列处理 + 合同 ID 解析 helpers。 v1.1.31:从 platform_collector.py 拆分出来的队列处理子模块。 包含 7 个函数: - _download_materials_for_row(点续聘材料包按钮抓卡片文本) - _find_row_for_contract(在 hr/contracts 表格找 contract_id 行号) - process_queue(领取 download_tasks 队列、登录、抓材料、落库) - _gen_contract_id / _normalize_date / _parse_days / _parse_alert(4 个纯解析 helper) 设计要点(保持 v1.1.30 不变): - 登录子流程通过 service.platform_collector_login 顶部 import - facade 持有的符号(_goto / _is_logged_in / 异常类 / SPA_LOGIN_URL 等) 通过 `from service import platform_collector as _pc` lazy import - 4 个 helper 函数(_gen_contract_id / _normalize_date / _parse_days / _parse_alert) 在本模块内自包含,collect / persist / queue 都可能用到(通过 facade re-export 访问) """ from __future__ import annotations import re import sqlite3 from datetime import datetime from db import repository as repo from db.connection import get_connection from db.migrations import ensure_materials_content_schema from util.constants import ( CARD_APPEAR_WAIT_MS, DEFAULT_PAGE_TIMEOUT_MS, HUMAN_WAIT_TIMEOUT, POST_LOGIN_SETTLE_MS, SPA_HOME_RETRY_DELAY_MS, SPA_HOME_RETRY_MAX, SPA_HYDRATION_MS, TABLE_INITIAL_WAIT_MS, TABLE_POLL_INTERVAL_MS, TABLE_POLL_MAX_ATTEMPTS, TABLE_TR_TIMEOUT_MS, ) from util.logging_setup import get_skill_logger from service.platform_collector_login import ( _fill_spa_login, _wait_for_manual_login, _wait_for_url_stable, ) logger = get_skill_logger() def _download_materials_for_row( page, row_idx: int, contract_id: str, mat_btn_count: int, ) -> list[dict]: """点续聘材料包按钮抓卡片文本(与 v1.1.17 collect_all 内联逻辑等价)。 返回 materials 列表(kind=sandbox_card,含 file_name + text)。 """ if mat_btn_count == 0: return [] materials: list[dict] = [] try: btn = page.locator("table tbody tr").nth(row_idx).locator( "button:has-text('续聘材料包')" ) btn.click() page.wait_for_timeout(CARD_APPEAR_WAIT_MS) cards_text = page.locator( "main .grid > div, main > div > div > div" ).all_inner_texts() materials_text = "\n\n".join(t.strip() for t in cards_text if t.strip()) if materials_text: materials.append({ "kind": "sandbox_card", "file_name": f"{contract_id}_materials.txt", "text": materials_text, }) logger.info( "platform_collector: 抓到卡片文本 %d 字 (%s)", len(materials_text), contract_id, ) except Exception as e: logger.warning("抓卡片失败 %s: %s", contract_id, e) return materials def _find_row_for_contract(page, contract_id: str) -> int | None: """在 hr/contracts 表格中找 contract_id 所在行号。 支持 7 列(用 teacher_name 派生 contract_id)与 10 列(用 emp_id)。 不匹配返回 None(调用方 mark_task_failed)。 """ rows = page.eval_on_selector_all( "table tbody tr", "rs => rs.map(r => Array.from(r.querySelectorAll('td')).map(td => td.innerText))", ) for i, r in enumerate(rows): ncols = len(r) if ncols < 6: continue first_col = r[0].strip() is_emp_id = bool(re.match(r"^[A-Za-z]?\d+$", first_col)) if is_emp_id and ncols >= 9: emp_id, contract_end = first_col, r[5].split("~")[-1].strip() elif ncols >= 6: emp_id, contract_end = first_col, r[2].strip() else: continue candidate = _gen_contract_id(emp_id, contract_end) if candidate == contract_id: return i return None def process_queue( limit: int = 5, headless: bool = True, human_wait_timeout: int = HUMAN_WAIT_TIMEOUT, ids_filter: str = "", ) -> dict: """从 download_tasks 队列取 pending 任务,对每个合同重新走材料抓取。 与 collect_all 重复登录流程,但每行只点「续聘材料包」按钮抓卡片,不重新抓合同表。 参数: limit: 本次最多处理任务数(0 = 不限) ids_filter: 逗号分隔 task_id 列表(只处理这些 pending) headless: 无头模式(需要 profile cookie 已保存) human_wait_timeout: 手工登录 fallback 等待秒数 返回 {processed, succeeded, failed, details}。 """ # Lazy import: 读 facade 持有的 _safe_getenv_dir / _goto / _is_logged_in / 异常类 / URL 常量 from service import platform_collector as _pc try: from playwright.sync_api import sync_playwright except ImportError: logger.error("playwright 未安装,跳过队列处理") return {"processed": 0, "succeeded": 0, "failed": 0, "details": [], "error": "playwright_not_installed"} # 1. 领取一批任务 ids_list = [s.strip() for s in ids_filter.split(",") if s.strip()] if ids_filter else None conn = get_connection() try: tasks = repo.claim_pending_tasks( conn, limit=limit if limit > 0 else 999, ids_filter=ids_list, ) finally: conn.close() if not tasks: return {"processed": 0, "succeeded": 0, "failed": 0, "details": []} profile_dir = _pc._safe_getenv_dir("SMARTCAMPUS_PROFILE_DIR", "playwright-profile") download_dir = _pc._safe_getenv_dir("MATERIALS_DOWNLOAD_DIR", "materials") profile_dir.mkdir(parents=True, exist_ok=True) download_dir.mkdir(parents=True, exist_ok=True) details: list[dict] = [] succeeded = 0 failed = 0 with sync_playwright() as p: ctx = p.chromium.launch_persistent_context( user_data_dir=str(profile_dir), headless=headless, accept_downloads=True, viewport={"width": 1440, "height": 900}, channel="msedge" if not headless else None, ) if not headless else p.chromium.launch_persistent_context( user_data_dir=str(profile_dir), headless=True, accept_downloads=True, viewport={"width": 1440, "height": 900}, ) try: page = ctx.new_page() page.set_default_timeout(DEFAULT_PAGE_TIMEOUT_MS) # 登录:复用 v1.1.17 collect_all 的探针 + 自动 fill + 手工 fallback 流程 logger.info("platform_collector.process_queue: 启动 profile=%s headless=%s 待处理 %d", profile_dir, headless, len(tasks)) _pc._goto(page, _pc.SPA_HOME_URL) page.wait_for_timeout(SPA_HYDRATION_MS) if _pc._is_logged_in(page): logger.info("platform_collector.process_queue: ✅ 已登录态") else: _pc._goto(page, _pc.SPA_LOGIN_URL) page.wait_for_timeout(SPA_HYDRATION_MS) if not _fill_spa_login(page): if not headless: _wait_for_manual_login(page, wait_sec=human_wait_timeout) else: raise _pc.LoginTimeoutError("process_queue headless 自动 fill 失败") else: page.wait_for_timeout(POST_LOGIN_SETTLE_MS) # 强制 goto SPA_HOME + 侧边栏 hr/contracts for _ in range(SPA_HOME_RETRY_MAX): if _pc._is_logged_in(page): break _pc._goto(page, _pc.SPA_HOME_URL) page.wait_for_timeout(SPA_HOME_RETRY_DELAY_MS) if _fill_spa_login(page): page.wait_for_timeout(POST_LOGIN_SETTLE_MS) if not _pc._is_logged_in(page): raise _pc.LoginTimeoutError("process_queue 仍未登录") # 等表格 page.wait_for_timeout(TABLE_INITIAL_WAIT_MS) found_table = False for _attempt in range(TABLE_POLL_MAX_ATTEMPTS): page.wait_for_timeout(TABLE_POLL_INTERVAL_MS) try: page.wait_for_selector( "table tbody tr", timeout=TABLE_TR_TIMEOUT_MS, ) found_table = True break except Exception: pass if not found_table: raise _pc.TableNotFoundError("process_queue 表格未找到") mat_btn_count = page.locator( "table tbody tr button:has-text('续聘材料包')" ).count() # 2. 逐任务处理 for t in tasks: contract_id = t["contract_id"] task_id = t["task_id"] conn = get_connection() try: row_idx = _find_row_for_contract(page, contract_id) if row_idx is None: repo.mark_task_failed(conn, task_id, "contract_not_found_in_table") failed += 1 details.append({"task_id": task_id, "contract_id": contract_id, "error": "contract_not_found_in_table"}) conn.commit() continue materials = _download_materials_for_row( page, row_idx, contract_id, mat_btn_count, ) ensure_materials_content_schema(conn) for m in materials: if isinstance(m, dict) and m.get("kind") == "sandbox_card": conn.execute( """ INSERT OR REPLACE INTO materials_content (contract_id, file_name, file_path, extracted_text, parse_method, parsed_at) VALUES (?, ?, ?, ?, 'sandbox_card', ?) """, ( contract_id, m["file_name"], f"sandbox://hr/contracts/{contract_id}", m["text"], datetime.now().isoformat(timespec="seconds"), ), ) repo.parse_sandbox_card_performance(conn, contract_id, m["text"]) result_path = ( f"sandbox://hr/contracts/{contract_id}" if materials else "no_materials_captured" ) repo.mark_task_success(conn, task_id, result_path) succeeded += 1 details.append({ "task_id": task_id, "contract_id": contract_id, "ok": True, "materials_count": len(materials), }) conn.commit() except Exception as e: repo.mark_task_failed(conn, task_id, str(e)) failed += 1 details.append({ "task_id": task_id, "contract_id": contract_id, "error": str(e), }) conn.commit() logger.warning("process_queue: task=%s 失败: %s", task_id, e) finally: conn.close() finally: ctx.close() return { "processed": len(tasks), "succeeded": succeeded, "failed": failed, "details": details, } def _gen_contract_id(emp_id: str, end_date: str) -> str: """从 (工号, 到期日) 派 contract_id,确保幂等。 sandbox v1.1.1 起改用「工号」作主键(之前用姓名)。 """ safe_emp = re.sub(r"[^\w]", "", emp_id)[:10] or "T" safe_end = end_date.replace("-", "")[:8] return f"RPA-{safe_emp}-{safe_end}" def _normalize_date(s: str) -> str: """尽量把 sandbox 返回的日期归一为 YYYY-MM-DD。""" s = s.strip() if re.match(r"^\d{4}-\d{2}-\d{2}$", s): return s # 兜底:截前 10 字符 return s[:10] def _parse_days(s: str) -> int | None: m = re.search(r"-?\d+", s) return int(m.group(0)) if m else None def _parse_alert(s: str) -> str: s = s.strip() if "红" in s: return "red" if "橙" in s: return "orange" if "黄" in s: return "yellow" return "unknown"