"""Playwright RPA:SmartCampus HR 平台合同台账采集主流程。 v1.1.31:从 platform_collector.py 拆分出来的 collect_all 子模块。 包含 2 个函数: - collect_all(登录 + 抓 hr/contracts 表格 + 落库 + 入队下载任务) - _error_result(错误返回包装) 设计要点(保持 v1.1.30 不变): - 登录子流程通过 service.platform_collector_login 顶部 import(login 是叶子模块,无 cycle) - facade 持有的符号(_goto / _dump_page / _is_logged_in / 异常类 / PLATFORM_URL 等) 通过 `from service import platform_collector as _pc` lazy import → 避免 facade ↔ collect cycle;测试 patch facade attr 后立即生效 - persist / queue 持有的函数(_persist_contracts / _gen_contract_id 等)仍位于 facade, Stage 3/4 移到 persist/queue 子模块后,collect 改为对应模块顶部 import """ from __future__ import annotations import time from datetime import datetime from util.constants import ( CARD_APPEAR_WAIT_MS, DEFAULT_PAGE_TIMEOUT_MS, FULL_RENDER_WAIT_MS, HUMAN_WAIT_TIMEOUT, POST_LOGIN_SETTLE_MS, POST_RELOAD_LOGIN_HYDRATION_MS, POST_RELOGIN_GOTO_WAIT_MS, PUSHSTATE_SETTLE_MS, RELOAD_DOMCONTENTLOADED_TIMEOUT_MS, SIDEBAR_CLICK_URL_TIMEOUT_MS, SIDEBAR_LINK_TIMEOUT_MS, SPA_HYDRATION_MS, TABLE_INITIAL_WAIT_MS, TABLE_POLL_INTERVAL_MS, TABLE_POLL_MAX_ATTEMPTS, TABLE_TR_TIMEOUT_MS, ) from util.logging_setup import get_skill_logger from service.platform_collector_login import ( _fill_spa_login, _fill_spa_login_from_env, _wait_for_login_inputs_filled, _wait_for_manual_login, _wait_for_url_stable, ) logger = get_skill_logger() def collect_all( limit: int = 0, headless: bool = True, human_wait_timeout: int = HUMAN_WAIT_TIMEOUT, ) -> dict: """登录 sandbox 平台 → 抓合同表 + 材料 → 落库。 human_wait_timeout: 手工登录 fallback 等待秒数(默认 HUMAN_WAIT_TIMEOUT=180)。 失败时可能抛出 RPAError(LoginTimeoutError / MissingBrowserError / TableNotFoundError)。 """ # Lazy import: 读 facade 持有的 _safe_getenv_dir / 异常类 / PLATFORM_URL 等 from service import platform_collector as _pc try: from playwright.sync_api import sync_playwright except ImportError: logger.error("playwright 未安装,跳过 RPA 采集") return _error_result("playwright_not_installed", 0) profile_dir = _pc._safe_getenv_dir("SMARTCAMPUS_PROFILE_DIR", "playwright-profile") download_dir = _pc._safe_getenv_dir("MATERIALS_DOWNLOAD_DIR", "materials") profile_dir.mkdir(parents=True, exist_ok=True) download_dir.mkdir(parents=True, exist_ok=True) logger.info( "platform_collector: 启动 profile=%s download_dir=%s headless=%s limit=%d " "human_wait_timeout=%d", profile_dir, download_dir, headless, limit, human_wait_timeout, ) contracts: list[dict] = [] fetched_rows = 0 downloaded = 0 with sync_playwright() as p: # v1.1.9:headed 模式优先用系统 Edge(跟 Windows 桌面 session 同源, # 能真正弹窗让你看到并手工登录);找不到则 fallback Playwright 内置 chromium if not headless: try: ctx = p.chromium.launch_persistent_context( user_data_dir=str(profile_dir), headless=False, accept_downloads=True, channel="msedge", viewport={"width": 1440, "height": 900}, ) logger.info( "platform_collector: 使用系统 Edge(headed 弹窗)" ) except Exception as e: logger.warning( "platform_collector: 系统 Edge 启动失败(%s),fallback Playwright chromium", e, ) ctx = p.chromium.launch_persistent_context( user_data_dir=str(profile_dir), headless=False, accept_downloads=True, viewport={"width": 1440, "height": 900}, ) else: ctx = p.chromium.launch_persistent_context( user_data_dir=str(profile_dir), headless=True, accept_downloads=True, viewport={"width": 1440, "height": 900}, ) try: page = ctx.new_page() # v1.1.6:覆盖 Playwright 默认操作超时(之前无 set_default_timeout,导致 # 所有 wait_for_selector 默认 30s,调试时不直观) page.set_default_timeout(DEFAULT_PAGE_TIMEOUT_MS) # 1. v1.1.21:先 goto **根域**(sandbox 登录页),让 SPA hydrate 2 秒 logger.info( "platform_collector: 启动,goto ROOT_URL=%s (当前 URL=%s)", _pc.ROOT_URL, page.url, ) _pc._goto(page, _pc.ROOT_URL) # 给 React/Vue 框架 2 秒 hydrate 时间,看登录 input 是否就绪 page.wait_for_timeout(2_000) if _pc._is_logged_in(page): logger.info( "platform_collector: ✅ 已登录态(URL=%s),跳过登录流程", page.url, ) else: # 2. 等 #jc-user / #jc-pass 出现(最多 2 秒) # 若 sandbox 已预填 demo/123456 → 自动点 Sign in # 若 input 为空 → 等用户输入(**v1.1.24 用户自己点登录按钮**) auto_clicked = _pc._try_auto_sign_in(page, wait_input_sec=2) if auto_clicked: logger.info( "platform_collector: ✅ 自动登录成功(预填 demo),等 URL 跳到 /app/" ) elif not headless: # 3. input 为空:等用户输入 → **用户自己点 Sign in**(脚本不抢按钮) if _wait_for_login_inputs_filled(page, wait_sec=human_wait_timeout): logger.info( "platform_collector: ✅ 用户完成输入并跳转,等 URL 跳到 /app/" ) else: # 超时未跳转 → fallback 到原来的多信号等待 logger.warning( "platform_collector: 用户未跳转,fallback 到 _wait_for_manual_login" ) _wait_for_manual_login(page, wait_sec=human_wait_timeout) else: raise _pc.LoginTimeoutError( f"headless 模式下未登录,且无法手工登录" f"(请用 --show-browser 重试)" ) # 4. v1.1.27:等 URL 跳到任一「SPA 已路由」状态(最多 30s,**不做任何操作**) # - /app/ = 已到 SPA 主页 # - /select = sandbox 中间页 # - /login 或 /signin = sandbox 又把人踢回登录页(根域登录后常见, # sandbox 会跳到 /industries/campus/smartcampus/login SPA 二级登录页) # - 如果 URL 跳到 login/signin,**立即**退出 30 秒等待(不再干等), # 让 4.5 处理。否则会等满 30 秒才发现要进 SPA 登录兜底,造成卡顿。 spa_url_seen = False if not _wait_for_url_stable( page, ("/app/", "/select", "/login", "/signin"), max_wait_sec=30, ): logger.warning( "platform_collector: 登录后 30s 内 URL 未稳定到 /app/、/select、" "/login 或 /signin(当前 URL=%s)", page.url, ) # 4.5. v1.1.28/31:URL 落在 SPA 二级登录页 → 主动等 input hydrate(最多 10 秒) # → 从 .env 读凭证 fill(v1.1.31:密码不硬编码)→ 点登录 → 等 URL 跳离 # v1.1.27 失败的根因:URL 已稳定在 /login 时,React SPA input 可能还没渲染, # 找不到 input → False → 卡 180 秒等用户输入。 # 修复:先等 input 真正 hydrate 出来,再 fill+click。 cur_url_after_stable = page.url.lower() if "/login" in cur_url_after_stable or "/signin" in cur_url_after_stable: logger.warning( "platform_collector: 登录后落在 SPA 二级登录页(%s)," "v1.1.31 触发 SPA 登录兜底(等 input hydrate + 从 .env 读凭证 fill)", page.url, ) # 1. 等 input 真正 hydrate 出来(最多 10 秒) try: page.wait_for_selector( "#jc-user, #jc-pass, input[type='tel'], input[type='password']", timeout=10_000, state="attached", ) page.wait_for_timeout(500) # 让 React 状态稳定 except Exception as e: logger.warning( "platform_collector: SPA 登录页 10s 内未渲染 input: %s", e, ) # 2. 从 .env 读凭证 fill + 点登录 spa_filled = _fill_spa_login_from_env(page) if spa_filled: logger.info( "platform_collector: ✅ SPA 二级登录页(来自 .env)已自动填好并点登录," "等 URL 跳到 /app/ 或 /select" ) # 等 URL 跳离 login(30 秒内跳到 /app/ 或 /select 都算成功) if not _wait_for_url_stable(page, ("/app/", "/select"), max_wait_sec=30): logger.warning( "platform_collector: SPA 登录后 30s 内 URL 未稳定到 /app/或/select" "(当前 URL=%s,.env 凭证可能失效)", page.url, ) # 凭证失效:等用户手工登录(headless 模式直接抛错) if not headless: _wait_for_manual_login(page, wait_sec=human_wait_timeout) else: raise _pc.LoginTimeoutError( f"SPA 二级登录页 .env 凭证 fill 后 URL 不跳离" f"(可能凭证失效,URL={page.url})" ) else: # .env 无凭证 / input 仍未渲染 → 等用户手工输入 logger.warning( "platform_collector: SPA 登录页自动 fill 失败" "(.env 无凭证或 input 未渲染),fallback 等用户输入" ) if not headless: if _wait_for_login_inputs_filled(page, wait_sec=human_wait_timeout): _wait_for_url_stable(page, ("/app/", "/select"), max_wait_sec=30) else: _wait_for_manual_login(page, wait_sec=human_wait_timeout) else: raise _pc.LoginTimeoutError( f"SPA 二级登录页 .env 无凭证且 headless 无法手工登录" f"(URL={page.url})" ) # 5. v1.1.22:显式跳到 SPA_HOME_URL(/app/home),等侧边栏 link 出现 # 避免 sandbox 在 /select 中间页时直接找 sidebar link 找不到 # v1.1.32 优化:URL 不在 /app/home(可能在 /select 中间页)就 goto(5s 等) if "/app/home" not in page.url.lower(): logger.info( "platform_collector: 当前 URL=%s,跳到 SPA_HOME_URL=%s", page.url, _pc.SPA_HOME_URL, ) _pc._goto(page, _pc.SPA_HOME_URL) page.wait_for_timeout(SPA_HYDRATION_MS) # v1.1.32 优化:等 URL 稳定到 /app/home(最多 5s,比 15s 快很多) if not _wait_for_url_stable(page, "/app/home", max_wait_sec=5): logger.warning( "platform_collector: 5s 内未稳定到 /app/home(当前 URL=%s)", page.url, ) # 6. 等 SPA 渲染 + click 侧边栏 hr/contracts(SPA 路由,避免 goto 触发整页 reload 丢 session) try: # dump SPA home 一次(用于调试 selector) page.wait_for_timeout(POST_LOGIN_SETTLE_MS) _pc._dump_page(page, download_dir.parent / "spa_home_dump.html", "SPA home") # 等 SPA 真正加载完(侧边栏 link 出现,最长 SIDEBAR_LINK_TIMEOUT_MS) try: page.wait_for_selector( "a[href$='/hr/contracts']", timeout=SIDEBAR_LINK_TIMEOUT_MS, state="attached", ) except Exception as e: logger.warning("等侧边栏 hr/contracts link 超时: %s", e) link = page.locator("a[href$='/hr/contracts']").first if link.count() > 0: link.click() logger.info("platform_collector: click 侧边栏 hr/contracts") try: page.wait_for_url( lambda u: "/hr/contracts" in u, timeout=SIDEBAR_CLICK_URL_TIMEOUT_MS, ) except Exception as e: logger.warning("等 /hr/contracts URL 超时: %s;当前 URL=%s", e, page.url) else: logger.info( "platform_collector: 侧边栏 link 仍未出现," "使用 SPA 路由(pushState)到 hr/contracts, 当前 URL=%s", page.url, ) # 用 SPA 内部路由而非整页 goto,避免 session 被 server 端踢回 login try: page.evaluate( "u => history.pushState({}, '', u) || " "window.dispatchEvent(new PopStateEvent('popstate'))," " '/industries/campus/smartcampus/app/hr/contracts'" ) page.wait_for_timeout(PUSHSTATE_SETTLE_MS) # v1.1.32:pushState 后 URL 可能没切(sandbox mock 偶发), # 检查 URL 真到 /hr/contracts 才算成功,否则 fallback goto if "/hr/contracts" not in page.url.lower(): logger.warning( "platform_collector: pushState 后 URL=%s 未到 /hr/contracts," "fallback goto PLATFORM_URL", page.url, ) _pc._goto(page, _pc.PLATFORM_URL) except Exception as e: logger.warning("SPA pushState 失败: %s", e) _pc._goto(page, _pc.PLATFORM_URL) except Exception as e: logger.warning("click 侧边栏失败: %s;fallback goto", e) _pc._goto(page, _pc.PLATFORM_URL) logger.info("platform_collector: 当前 URL=%s", page.url) # 5.5. v1.1.8:**首次登录后不再强制刷新**,避免打断用户手工登录态。 # 改为「等表格出现」兜底:给 SPA 一个初始等待窗口(TABLE_INITIAL_WAIT_MS*2), # 如果表格仍未出现,**才**刷新一次(sandbox mock 异步加载场景)。 logger.info( "platform_collector: 进入 hr/contracts 后**不刷新**,等待表格出现" "(首次登录态不要打断)" ) # 5.6. 兜底刷新:仅当等表格超时后才触发;刷新后可能被踢回 login,需重新登录 # 修复:之前没检查 URL,刷新后停在 login 页继续往后走,找不到表格就 dump 报错 # v1.1.32 优化:刷回 login 时用 _fill_spa_login_from_env(从 .env 读), # 不再调硬编码 demo/123456 的 _fill_spa_login(与 .env 凭证一致)。 def _try_reload_after_table_timeout(): logger.warning( "platform_collector: 表格未在 %dms 内出现,触发兜底刷新", TABLE_INITIAL_WAIT_MS * 2, ) try: page.reload( wait_until="networkidle", timeout=RELOAD_NETWORKIDLE_TIMEOUT_MS, ) except Exception as e: logger.warning("刷新 networkidle 失败: %s;尝试 domcontentloaded", e) try: page.reload( wait_until="domcontentloaded", timeout=RELOAD_DOMCONTENTLOADED_TIMEOUT_MS, ) except Exception as e2: logger.warning("刷新 domcontentloaded 也失败: %s", e2) if "/login" in page.url.lower() or "/signin" in page.url.lower(): logger.warning( "platform_collector: 刷新后回到登录页(%s),重新登录", page.url, ) # v1.1.32:先等 input hydrate,再走 .env 凭证 fill try: page.wait_for_selector( "#jc-user, #jc-pass, input[type='tel'], input[type='password']", timeout=10_000, state="attached", ) page.wait_for_timeout(500) except Exception as e: logger.warning( "platform_collector: 刷新后 login 页 10s 内未渲染 input: %s", e, ) if not _fill_spa_login_from_env(page): # v1.1.32:.env 无凭证时 fallback 到硬编码 demo(保持 v1.1.10 兼容) logger.warning( "platform_collector: .env 无凭证,fallback 硬编码 demo/123456" ) if not _fill_spa_login(page): if not headless: _wait_for_manual_login(page, wait_sec=human_wait_timeout) else: raise _pc.LoginTimeoutError( "刷新后回到 login 页且 headless 无法手工登录" ) else: page.wait_for_timeout(POST_LOGIN_SETTLE_MS) else: # .env 凭证 fill 成功 → 等 URL 跳离 login _wait_for_url_stable(page, ("/app/", "/select"), max_wait_sec=15) _pc._goto(page, _pc.PLATFORM_URL) page.wait_for_timeout(POST_RELOAD_LOGIN_HYDRATION_MS) # 7. dump 当前 hr/contracts 页面(用于调试) _pc._dump_page(page, download_dir.parent / "hr_contracts_dump.html", "hr/contracts") # 8. 等待表格加载(v1.1.8:先等,**表格超时未出现才触发兜底刷新**—— # 修复「首次登录后强制刷新打断手工登录态」问题) page.wait_for_timeout(TABLE_INITIAL_WAIT_MS) # 给 React 初始化 + mock 数据填充时间 found_table = False for attempt in range(TABLE_POLL_MAX_ATTEMPTS): page.wait_for_timeout(TABLE_POLL_INTERVAL_MS) cur_url = page.url try: page.wait_for_selector( "table tbody tr", timeout=TABLE_TR_TIMEOUT_MS, ) found_table = True break except Exception: logger.info( "platform_collector: URL=%s 等待表格加载 (attempt %d/%d)...", cur_url, attempt + 1, TABLE_POLL_MAX_ATTEMPTS, ) if not found_table: # v1.1.8:表格没出现,**先尝试兜底刷新一次**(sandbox mock 异步加载场景); # 刷新后再次轮询表格,仍找不到才报 TableNotFoundError。 logger.warning( "platform_collector: 首次轮询未找到表格,触发兜底刷新(sandbox mock 异步加载场景)" ) _try_reload_after_table_timeout() # 刷新后重新轮询一次(最多 5 次) for attempt2 in range(5): page.wait_for_timeout(TABLE_POLL_INTERVAL_MS) try: page.wait_for_selector( "table tbody tr", timeout=TABLE_TR_TIMEOUT_MS, ) found_table = True logger.info( "platform_collector: 兜底刷新后找到表格 (attempt %d)", attempt2 + 1 ) break except Exception: logger.info( "platform_collector: 兜底刷新后仍等待表格 (attempt %d/5)...", attempt2 + 1, ) if not found_table: _pc._dump_page(page, download_dir.parent / "sandbox_dump.html", "sandbox") logger.warning("platform_collector: 表格未找到(兜底刷新后仍未出现)") raise _pc.TableNotFoundError( f"{TABLE_POLL_MAX_ATTEMPTS * TABLE_POLL_INTERVAL_MS // 1000}s " f"内未找到表格(兜底刷新后仍未出现) (URL={page.url})" ) # 表格已找到,等 tbody 完全填充 + dump 一份完整 DOM(用于诊断按钮 selector) try: page.wait_for_function( "document.querySelectorAll('table tbody tr').length > 0", timeout=DEFAULT_PAGE_TIMEOUT_MS, ) page.wait_for_timeout(FULL_RENDER_WAIT_MS) _pc._dump_page( page, download_dir.parent / "hr_contracts_full_dump.html", "完整 hr/contracts", ) except Exception as e: logger.warning("等完整渲染超时: %s", e) rows = page.eval_on_selector_all( "table tbody tr", "rs => rs.map(r => Array.from(r.querySelectorAll('td')).map(td => td.innerText))", ) fetched_rows = len(rows) logger.info("platform_collector: 抓到 %d 行", fetched_rows) # 检查每行是否有「续聘材料包」按钮 try: mat_btn_count = page.locator( "table tbody tr button:has-text('续聘材料包')" ).count() logger.info( "platform_collector: 「续聘材料包」按钮数=%d", mat_btn_count, ) except Exception: mat_btn_count = 0 # 抓「材料包」card 区段的 DOM(用于后续解析每行 click 后的卡片文本) # sandbox 行为:click 按钮后会在表格下方追加「续聘准备任务书」+「单位清单」卡片 for i, r in enumerate(rows): if limit and i >= limit: break # v1.1.8:表格列数自适应 —— sandbox 首次登录未刷新是 7 列 # (姓名/单位/到期日/剩余天数/预警/建议/按钮),刷新后才是 10 列 # (多 工号/岗位/合同类型/起止日期)。两种都支持。 ncols = len(r) if ncols < 6: logger.warning( "行 %d 列数=%d < 6,跳过: %s", i, ncols, r ) continue # 列序探测:判断首列是不是工号(纯数字 + 字母,纯姓名是中文) first_col = r[0].strip() import re as _re is_emp_id = bool(_re.match(r"^[A-Za-z]?\d+$", first_col)) if is_emp_id and ncols >= 9: # 10 列布局(含工号):0 工号 1 姓名 2 单位 3 岗位 4 合同类型 # 5 起止(含~) 6 剩余天数 7 预警 8 建议 9 按钮 emp_id = first_col teacher = r[1].strip() unit = r[2].strip() period = r[5].strip() contract_end = period.split("~")[-1].strip() if "~" in period else period days_raw = r[6].strip() alert_label = r[7].strip() suggestion_hint = r[8].strip() elif ncols >= 6: # 7 列布局(无工号):0 姓名 1 单位 2 到期日 3 剩余天数 4 预警 5 建议 6 按钮 teacher = first_col unit = r[1].strip() contract_end = r[2].strip() days_raw = r[3].strip() alert_label = r[4].strip() suggestion_hint = r[5].strip() emp_id = "" # 工号缺失,留空让 _gen_contract_id 用 teacher_name 兜底 else: logger.warning( "行 %d 列数=%d 无法识别布局,跳过: %s", i, ncols, r ) continue # 用工号生成 contract_id(稳定,不会随姓名变化) # v1.1.8:7 列布局无工号 → 用 teacher_name 派生(避免与 10 列布局冲突) if emp_id: contract_id = _pc._gen_contract_id(emp_id, contract_end) else: contract_id = _pc._gen_contract_id(teacher, contract_end) contract = { "contract_id": contract_id, "teacher_name": teacher, "unit_name": unit, "contract_end": _pc._normalize_date(contract_end), "days_to_expire_hint": _pc._parse_days(days_raw), "alert_level_hint": _pc._parse_alert(alert_label), "ai_suggestion_hint": suggestion_hint, "materials": [], } # 点「续聘材料包」按钮 → 等卡片展开 → 抓卡片文本(含 checklist) if mat_btn_count > 0: try: btn = page.locator("table tbody tr").nth(i).locator( "button:has-text('续聘材料包')" ) btn.click() # 等卡片出现(页面下方追加 h2「续聘准备任务书」/「单位清单」) page.wait_for_timeout(CARD_APPEAR_WAIT_MS) # 抓表格下方所有 card 文本 cards_text = page.locator( "main .grid > div, main > div > div > div" ).all_inner_texts() materials_text = "\n\n".join( t.strip() for t in cards_text if t.strip() ) if materials_text: # 存到 materials_content(sandbox mock 卡片文本作为材料包内容) contract["materials"].append({ "kind": "sandbox_card", "file_name": f"{contract_id}_materials.txt", "text": materials_text, }) downloaded += 1 logger.info( "platform_collector: 抓到卡片文本 %d 字 (%s)", len(materials_text), contract_id, ) # dump 一次(i==0)方便调试 if i == 0: _pc._dump_page( page, download_dir.parent / "after_click_dump.html", "首行 click 后", ) except Exception as e: logger.warning("抓卡片失败 %s: %s", contract_id, e) else: if i == 0: logger.info( "platform_collector: sandbox 未部署「续聘材料包」按钮," "跳过材料抓取(仅采集表格列)", ) contracts.append(contract) finally: ctx.close() saved = _pc._persist_contracts(contracts) logger.info( "platform_collector: contracts 落库完成 saved=%d / total_fetched=%d", saved, len(contracts), ) # v1.1.18:每个合同入队 download_tasks(独立事务,失败不影响主流程) contract_ids = [c["contract_id"] for c in contracts if c.get("contract_id")] enqueue_summary = _pc._enqueue_download_tasks_safe(contract_ids) return { "platform_url": _pc.PLATFORM_URL, "fetched_rows": fetched_rows, "saved_contracts": saved, "downloaded_materials": downloaded, "download_queue": enqueue_summary, "contracts": contracts, } def _error_result(reason: str, fetched: int) -> dict: # Lazy import: 读 facade 的 PLATFORM_URL from service import platform_collector as _pc return { "platform_url": _pc.PLATFORM_URL, "fetched_rows": fetched, "saved_contracts": 0, "downloaded_materials": 0, "contracts": [], "error": reason, }