"""Playwright RPA：从 SmartCampus HR 平台抓合同表 + 续聘材料包。 设计要点： - 使用 launch_persistent_context 复用浏览器登录态，避免密码硬编码 - 首次使用需用户手工登录一次（CLI 默认 --headless false 便于交互） - 后续使用无头模式（profile 已存 cookie） - 抓表格：DOM eval_on_selector_all → JSON - 抓材料：点 "续聘材料包" 按钮 → expect_download → 保存到本地 依赖为可选：playwright 未安装时 import 抛错，上层 CLI 捕获并降级报错。 导航流程（v1.1.1 强化版 — 强制 goto）： 1. goto 根域 2. 根域 login（如仍显示表单）→ 等跳到 /home 或 /select 3. 强制 goto /select 4. 在 /select 页面 click 智慧校园 tile → 等 URL 含 /industries/campus 5. 若 URL 含 /login，填 sandbox SPA 登录表单 → 等跳走 6. 强制 goto SPA_HOME 7. click 侧边栏 hr/contracts 8. 等 table tbody tr """ from __future__ import annotations import os import re import sqlite3 from datetime import datetime from pathlib import Path from db import repository as repo from util.constants import SKILL_SLUG from util.logging_setup import get_skill_logger from util.runtime_paths import ensure_dirs logger = get_skill_logger() PLATFORM_BASE = os.environ.get("SMARTCAMPUS_BASE_URL", "https://sandbox.jc2009.com") PLATFORM_HR_PATH = os.environ.get( "SMARTCAMPUS_HR_PATH", "/industries/campus/smartcampus/app/hr/contracts" ) PLATFORM_URL = PLATFORM_BASE + PLATFORM_HR_PATH ROOT_URL = PLATFORM_BASE + "/" SELECT_URL = PLATFORM_BASE + "/select" SPA_HOME_URL = PLATFORM_BASE + "/industries/campus/smartcampus/app/home" SPA_LOGIN_URL = PLATFORM_BASE + "/industries/campus/smartcampus/login" def _safe_getenv_dir(env_key: str, default_rel: str) -> Path: raw = os.environ.get(env_key) if raw: return Path(raw).resolve() from util.runtime_paths import get_data_dir return (get_data_dir() / default_rel).resolve() def _goto(page, url: str, timeout: int = 20_000) -> None: """带 networkidle / domcontentloaded 双兜底的 goto。""" try: page.goto(url, wait_until="networkidle", timeout=timeout) except Exception as e: logger.warning("goto networkidle 失败 %s: %s；尝试 domcontentloaded", url, e) try: page.goto(url, wait_until="domcontentloaded", timeout=timeout) except Exception as e2: logger.warning("goto domcontentloaded 也失败 %s: %s", url, e2) def _dump_page(page, path: Path, label: str) -> None: """Dump 当前页面 HTML 到指定路径，自动创建父目录。修复：父目录可能不存在导致静默失败。""" try: path.parent.mkdir(parents=True, exist_ok=True) html = page.content() path.write_text(html, encoding="utf-8") logger.info( "platform_collector: %s dump → %s (%d 字节)", label, path, len(html), ) except Exception as e: logger.warning("dump %s 失败: %s", label, e) def _fill_root_login(page) -> bool: """根域登录页（如还在）：fill demo/123456 → click Sign in。返回是否执行了登录。""" user_input = page.locator('#jc-user').first if user_input.count() == 0: return False user_input.fill("demo") page.locator('#jc-pass').first.fill("123456") page.locator('button[type="submit"]:has-text("Sign in")').first.click() logger.info("platform_collector: 根域自动登录 demo/123456...") try: page.wait_for_url( lambda u: "/home" in u or "/select" in u or "/industries/" in u, timeout=20_000, ) except Exception as e: logger.warning("等根域登录跳转超时: %s", e) logger.info("platform_collector: 根域登录后 URL=%s", page.url) return True def _click_campus_link(page) -> bool: """在 /select 页面点智慧校园 tile，返回是否点中。""" link = page.locator("a[href*='/industries/campus']").first if link.count() == 0: logger.warning("platform_collector: 未找到 智慧校园 link") return False link.click() logger.info("platform_collector: click 智慧校园 链接") try: page.wait_for_url(lambda u: "/industries/campus" in u, timeout=15_000) except Exception as e: logger.warning("等 sandbox 路径超时: %s；当前 URL=%s", e, page.url) return True def _fill_spa_login(page, settle_ms: int = 4000) -> bool: """sandbox SPA 登录（如还在登录页）：fill demo/123456 → click 登录 → 等跳转。 返回是否执行了登录（True = 已点登录按钮；调用方应继续等 URL 跳走）。 修复要点（登录时间太短导致采集不到数据）： - 默认 settle_ms 从 0 提到 4000ms，给 SPA 异步渲染 + token 写入 cookie 时间 - 等 username input 的 timeout 从 10s 提到 20s（mock 慢启动） - click 按钮后显式 wait_for_url 含更长 timeout（30s），URL 真的不含 /login 才返回 True - 多处过渡路径（/login, /signin, /auth/）一并判断，避免 SPA 中间态被误判为「已离开登录页」 """ url_lower = page.url.lower() if "/login" not in url_lower and "/signin" not in url_lower and "/auth/" not in url_lower: return False try: page.wait_for_selector('input[autocomplete="username"]', timeout=20_000) except Exception as e: logger.warning("SPA login input 未出现: %s；当前 URL=%s", e, page.url) return False spa_user = page.locator('input[autocomplete="username"]').first if spa_user.count() == 0: logger.warning("SPA login input count=0") return False spa_user.fill("demo") page.locator('input[autocomplete="current-password"]').first.fill("123456") page.locator( 'button:has-text("登录"), button[type="submit"]:has-text("Sign in")' ).first.click() logger.info("platform_collector: sandbox SPA 自动登录...") # click 后给 SPA 一点时间把 token 写入 cookie / 触发跳转 page.wait_for_timeout(settle_ms) # 再等到 URL 真的跳离 login 页（最长 30s；之前 20s 太短） try: page.wait_for_url( lambda u: "/login" not in u.lower() and "/signin" not in u.lower() and "/auth/" not in u.lower(), timeout=30_000, ) except Exception as e: logger.warning("SPA 登录超时: %s；当前 URL=%s", e, page.url) return False logger.info("platform_collector: SPA 登录后 URL=%s", page.url) return True def collect_all(limit: int = 0, headless: bool = True) -> dict: """登录 sandbox 平台 → 抓合同表 + 材料 → 落库。""" try: from playwright.sync_api import sync_playwright except ImportError: logger.error("playwright 未安装，跳过 RPA 采集") return _error_result("playwright_not_installed", 0) profile_dir = _safe_getenv_dir("SMARTCAMPUS_PROFILE_DIR", "playwright-profile") download_dir = _safe_getenv_dir("MATERIALS_DOWNLOAD_DIR", "materials") profile_dir.mkdir(parents=True, exist_ok=True) download_dir.mkdir(parents=True, exist_ok=True) logger.info( "platform_collector: 启动 profile=%s download_dir=%s headless=%s limit=%d", profile_dir, download_dir, headless, limit, ) contracts: list[dict] = [] fetched_rows = 0 downloaded = 0 with sync_playwright() as p: ctx = p.chromium.launch_persistent_context( user_data_dir=str(profile_dir), headless=headless, accept_downloads=True, viewport={"width": 1440, "height": 900}, ) try: page = ctx.new_page() # 1. 根域 goto + 根域登录（可选；root login 经常 demo/123456 失败） _goto(page, ROOT_URL) _fill_root_login(page) # 2. 直接 goto sandbox SPA login（绕过 /select 和 root 跳转） logger.info("platform_collector: 强制 goto sandbox SPA login, 当前 URL=%s", page.url) _goto(page, SPA_LOGIN_URL) # SPA hydration + framework 初始化时间（修复登录时间太短的核心点） page.wait_for_timeout(3000) # 3. sandbox SPA 登录（demo/123456 已知可用） # 等待 URL 真的跳离 login（settle_ms 已给到 4s，wait_for_url 又给 30s） if _fill_spa_login(page): # 等 SPA 路由稳定（之前这里没有 wait，直接 goto home 会打断 pending 跳转） page.wait_for_timeout(2000) # 4. 强制 goto SPA home（如果还停在 login，重试登录一次） for retry in range(2): if "/app/" in page.url and "/login" not in page.url.lower(): break logger.info( "platform_collector: 强制 goto SPA_HOME (retry=%d), 当前 URL=%s", retry, page.url, ) _goto(page, SPA_HOME_URL) page.wait_for_timeout(2500) if _fill_spa_login(page): page.wait_for_timeout(2000) # 5. 等 SPA 渲染 + click 侧边栏 hr/contracts（SPA 路由，避免 goto 触发整页 reload 丢 session） try: # dump SPA home 一次（用于调试 selector） page.wait_for_timeout(2000) _dump_page(page, download_dir.parent / "spa_home_dump.html", "SPA home") # 等 SPA 真正加载完（侧边栏 link 出现，最长 30s） # 修复：之前 15s 太短，sandbox mock 加载慢时直接 fallback 到 pushState → 表格加载不出来 try: page.wait_for_selector( "a[href$='/hr/contracts']", timeout=30_000, state="attached" ) except Exception as e: logger.warning("等侧边栏 hr/contracts link 超时: %s", e) link = page.locator("a[href$='/hr/contracts']").first if link.count() > 0: link.click() logger.info("platform_collector: click 侧边栏 hr/contracts") try: page.wait_for_url( lambda u: "/hr/contracts" in u, timeout=10_000 ) except Exception as e: logger.warning("等 /hr/contracts URL 超时: %s；当前 URL=%s", e, page.url) else: logger.info( "platform_collector: 侧边栏 link 仍未出现，" "使用 SPA 路由（pushState）到 hr/contracts, 当前 URL=%s", page.url, ) # 用 SPA 内部路由而非整页 goto，避免 session 被 server 端踢回 login try: page.evaluate( "u => history.pushState({}, '', u) || " "window.dispatchEvent(new PopStateEvent('popstate'))," " '/industries/campus/smartcampus/app/hr/contracts'" ) page.wait_for_timeout(1500) except Exception as e: logger.warning("SPA pushState 失败: %s", e) _goto(page, PLATFORM_URL) except Exception as e: logger.warning("click 侧边栏失败: %s；fallback goto", e) _goto(page, PLATFORM_URL) logger.info("platform_collector: 当前 URL=%s", page.url) # 5.5. 进入页面后刷新（SPA 进入新路由时 mock 数据可能未加载，需刷新触发） try: logger.info("platform_collector: 进入 hr/contracts 后刷新页面") page.reload(wait_until="networkidle", timeout=30_000) except Exception as e: logger.warning("刷新 networkidle 失败: %s；尝试 domcontentloaded", e) try: page.reload(wait_until="domcontentloaded", timeout=30_000) except Exception as e2: logger.warning("刷新 domcontentloaded 也失败: %s", e2) # 5.6. 刷新后可能被踢回 login（session 丢失），重新登录一次 # 修复：之前没检查 URL，刷新后停在 login 页继续往后走，找不到表格就 dump 报错 if "/login" in page.url.lower() or "/signin" in page.url.lower(): logger.warning( "platform_collector: 刷新后回到登录页（%s），重新登录", page.url, ) _goto(page, SPA_LOGIN_URL) page.wait_for_timeout(3000) if _fill_spa_login(page): page.wait_for_timeout(2000) _goto(page, PLATFORM_URL) page.wait_for_timeout(2000) # 7. dump 当前 hr/contracts 页面（用于调试） _dump_page(page, download_dir.parent / "hr_contracts_dump.html", "hr/contracts") # 8. 等待表格加载（SPA 异步填充 mock 数据，多等几秒） page.wait_for_timeout(3000) # 给 React 初始化 + mock 数据填充时间 found_table = False for attempt in range(20): page.wait_for_timeout(2000) cur_url = page.url try: page.wait_for_selector("table tbody tr", timeout=3000) found_table = True break except Exception: logger.info( "platform_collector: URL=%s 等待表格加载 (attempt %d/20)...", cur_url, attempt + 1, ) if not found_table: _dump_page(page, download_dir.parent / "sandbox_dump.html", "sandbox") logger.warning("platform_collector: 40s 后未找到表格") return _error_result("table_not_found_after_40s", 0) # 表格已找到，等 tbody 完全填充 + dump 一份完整 DOM（用于诊断按钮 selector） try: page.wait_for_function( "document.querySelectorAll('table tbody tr').length > 0", timeout=10_000, ) page.wait_for_timeout(1500) _dump_page( page, download_dir.parent / "hr_contracts_full_dump.html", "完整 hr/contracts", ) except Exception as e: logger.warning("等完整渲染超时: %s", e) rows = page.eval_on_selector_all( "table tbody tr", "rs => rs.map(r => Array.from(r.querySelectorAll('td')).map(td => td.innerText))", ) fetched_rows = len(rows) logger.info("platform_collector: 抓到 %d 行", fetched_rows) # 检查每行是否有「续聘材料包」按钮 try: mat_btn_count = page.locator( "table tbody tr button:has-text('续聘材料包')" ).count() logger.info( "platform_collector: 「续聘材料包」按钮数=%d", mat_btn_count, ) except Exception: mat_btn_count = 0 # 抓「材料包」card 区段的 DOM（用于后续解析每行 click 后的卡片文本） # sandbox 行为：click 按钮后会在表格下方追加「续聘准备任务书」+「单位清单」卡片 for i, r in enumerate(rows): if limit and i >= limit: break if len(r) < 7: logger.warning("行 %d 列数 < 7（新列序工号+姓名+单位+岗位+合同类型+起止+剩余+预警+建议），跳过: %s", i, r) continue # 新列序（sandbox 加了工号 / 岗位 / 合同类型 / 起止日期）： # 0: 工号 1: 姓名 2: 单位 3: 岗位 4: 合同类型 # 5: 起止日期 (含 ~ ) 6: 剩余天数 7: 预警 8: 建议（含原因） emp_id = r[0].strip() teacher = r[1].strip() unit = r[2].strip() position = r[3].strip() contract_type = r[4].strip() period = r[5].strip() # 起止日期形如 "2024-03-01 ~ 2026-04-15" → 取 ~ 后 contract_end = period.split("~")[-1].strip() if "~" in period else period days_raw = r[6].strip() alert_label = r[7].strip() suggestion_hint = r[8].strip() if len(r) > 8 else "" # 用工号生成 contract_id（稳定，不会随姓名变化） contract_id = _gen_contract_id(emp_id, contract_end) contract = { "contract_id": contract_id, "teacher_name": teacher, "unit_name": unit, "contract_end": _normalize_date(contract_end), "days_to_expire_hint": _parse_days(days_raw), "alert_level_hint": _parse_alert(alert_label), "ai_suggestion_hint": suggestion_hint, "materials": [], } # 点「续聘材料包」按钮 → 等卡片展开 → 抓卡片文本（含 checklist） if mat_btn_count > 0: try: btn = page.locator("table tbody tr").nth(i).locator( "button:has-text('续聘材料包')" ) btn.click() # 等卡片出现（页面下方追加 h2「续聘准备任务书」/「单位清单」） # 修复：1500ms 太短，sandbox mock 渲染卡片动画慢，导致抓到的文本不全 page.wait_for_timeout(3000) # 抓表格下方所有 card 文本 cards_text = page.locator( "main .grid > div, main > div > div > div" ).all_inner_texts() materials_text = "\n\n".join( t.strip() for t in cards_text if t.strip() ) if materials_text: # 存到 materials_content（sandbox mock 卡片文本作为材料包内容） contract["materials"].append({ "kind": "sandbox_card", "file_name": f"{contract_id}_materials.txt", "text": materials_text, }) downloaded += 1 logger.info( "platform_collector: 抓到卡片文本 %d 字 (%s)", len(materials_text), contract_id, ) # dump 一次（i==0）方便调试 if i == 0: _dump_page( page, download_dir.parent / "after_click_dump.html", "首行 click 后", ) except Exception as e: logger.warning("抓卡片失败 %s: %s", contract_id, e) else: if i == 0: logger.info( "platform_collector: sandbox 未部署「续聘材料包」按钮，" "跳过材料抓取（仅采集表格列）", ) contracts.append(contract) finally: ctx.close() saved = _persist_contracts(contracts) return { "platform_url": PLATFORM_URL, "fetched_rows": fetched_rows, "saved_contracts": saved, "downloaded_materials": downloaded, "contracts": contracts, } def _persist_contracts(contracts: list[dict]) -> int: """把抓到的合同 + 材料落到 SQLite。仅插入 contracts 表的占位行， materials_content 由 parse-materials CLI 后续解析写入。 真正的合同台账来自 HR 系统 API（生产）或样例 CSV（本地）， 本采集仅作为"快照"记录抓取时间与材料路径。 """ if not contracts: return 0 from db.connection import get_connection conn = get_connection() try: with conn: for c in contracts: # upsert contracts（仅更新与 sandbox 相关的字段） conn.execute( """ INSERT INTO contracts (contract_id, employee_id, teacher_name, unit_id, unit_name, position, contract_type, contract_start, contract_end, trial_period_end, status, last_material_id, renewed_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '可续聘', NULL, NULL) ON CONFLICT(contract_id) DO UPDATE SET teacher_name = excluded.teacher_name, unit_name = excluded.unit_name, contract_end = excluded.contract_end """, ( c["contract_id"], c["contract_id"], # employee_id 暂用 contract_id c["teacher_name"], "U-RPA", # unit_id 占位 c["unit_name"], "（RPA 采集）", "（RPA 采集）", "1970-01-01", c["contract_end"], c["contract_end"], ), ) # 写 materials_content（sandbox card 文本 + parse-materials 解析后文件路径） from db.migrations import ensure_materials_content_schema ensure_materials_content_schema(conn) for m in c["materials"]: if isinstance(m, dict) and m.get("kind") == "sandbox_card": # sandbox mock 卡片文本，直接写入 extracted_text conn.execute( """ INSERT OR REPLACE INTO materials_content (contract_id, file_name, file_path, extracted_text, parse_method, parsed_at) VALUES (?, ?, ?, ?, 'sandbox_card', ?) """, ( c["contract_id"], m["file_name"], f"sandbox://hr/contracts/{c['contract_id']}", m["text"], datetime.now().isoformat(timespec="seconds"), ), ) # 同步解析 → 抽 year + 等次 → 写入 performance_records perf_n = repo.parse_sandbox_card_performance( conn, c["contract_id"], m["text"], ) if perf_n > 0: logger.info( "platform_collector: %s 解析卡片 → performance %d 条", c["contract_id"], perf_n, ) else: # 本地文件路径（parse-materials 解析后写入） fp = m if isinstance(m, str) else m.get("file_path") fname = Path(fp).name if fp else "unknown" conn.execute( """ INSERT OR REPLACE INTO materials_content (contract_id, file_name, file_path, extracted_text, parse_method, parsed_at) VALUES (?, ?, ?, NULL, 'pending', ?) """, ( c["contract_id"], fname, fp, datetime.now().isoformat(timespec="seconds"), ), ) return len(contracts) finally: conn.close() def _error_result(reason: str, fetched: int) -> dict: return { "platform_url": PLATFORM_URL, "fetched_rows": fetched, "saved_contracts": 0, "downloaded_materials": 0, "contracts": [], "error": reason, } def _gen_contract_id(emp_id: str, end_date: str) -> str: """从 (工号, 到期日) 派 contract_id，确保幂等。 sandbox v1.1.1 起改用「工号」作主键（之前用姓名）。 """ safe_emp = re.sub(r"[^\w]", "", emp_id)[:10] or "T" safe_end = end_date.replace("-", "")[:8] return f"RPA-{safe_emp}-{safe_end}" def _normalize_date(s: str) -> str: """尽量把 sandbox 返回的日期归一为 YYYY-MM-DD。""" s = s.strip() if re.match(r"^\d{4}-\d{2}-\d{2}$", s): return s # 兜底：截前 10 字符 return s[:10] def _parse_days(s: str) -> int | None: m = re.search(r"-?\d+", s) return int(m.group(0)) if m else None def _parse_alert(s: str) -> str: s = s.strip() if "红" in s: return "red" if "橙" in s: return "orange" if "黄" in s: return "yellow" return "unknown"