626 lines
30 KiB
Python
626 lines
30 KiB
Python
"""Playwright RPA：SmartCampus HR 平台合同台账采集主流程。
|
||
|
||
v1.1.31：从 platform_collector.py 拆分出来的 collect_all 子模块。
|
||
包含 2 个函数：
|
||
- collect_all（登录 + 抓 hr/contracts 表格 + 落库 + 入队下载任务）
|
||
- _error_result（错误返回包装）
|
||
|
||
设计要点（保持 v1.1.30 不变）：
|
||
- 登录子流程通过 service.platform_collector_login 顶部 import（login 是叶子模块，无 cycle）
|
||
- facade 持有的符号（_goto / _dump_page / _is_logged_in / 异常类 / PLATFORM_URL 等）
|
||
通过 `from service import platform_collector as _pc` lazy import
|
||
→ 避免 facade ↔ collect cycle；测试 patch facade attr 后立即生效
|
||
- persist / queue 持有的函数（_persist_contracts / _gen_contract_id 等）仍位于 facade，
|
||
Stage 3/4 移到 persist/queue 子模块后，collect 改为对应模块顶部 import
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import time
|
||
from datetime import datetime
|
||
|
||
from util.constants import (
|
||
CARD_APPEAR_WAIT_MS,
|
||
DEFAULT_PAGE_TIMEOUT_MS,
|
||
FULL_RENDER_WAIT_MS,
|
||
HUMAN_WAIT_TIMEOUT,
|
||
POST_LOGIN_SETTLE_MS,
|
||
POST_RELOAD_LOGIN_HYDRATION_MS,
|
||
POST_RELOGIN_GOTO_WAIT_MS,
|
||
PUSHSTATE_SETTLE_MS,
|
||
RELOAD_DOMCONTENTLOADED_TIMEOUT_MS,
|
||
SIDEBAR_CLICK_URL_TIMEOUT_MS,
|
||
SIDEBAR_LINK_TIMEOUT_MS,
|
||
SPA_HYDRATION_MS,
|
||
TABLE_INITIAL_WAIT_MS,
|
||
TABLE_POLL_INTERVAL_MS,
|
||
TABLE_POLL_MAX_ATTEMPTS,
|
||
TABLE_TR_TIMEOUT_MS,
|
||
)
|
||
from util.logging_setup import get_skill_logger
|
||
from service.platform_collector_login import (
|
||
_fill_spa_login,
|
||
_fill_spa_login_from_env,
|
||
_wait_for_login_inputs_filled,
|
||
_wait_for_manual_login,
|
||
_wait_for_url_stable,
|
||
)
|
||
|
||
logger = get_skill_logger()
|
||
|
||
|
||
def collect_all(
|
||
limit: int = 0,
|
||
headless: bool = True,
|
||
human_wait_timeout: int = HUMAN_WAIT_TIMEOUT,
|
||
) -> dict:
|
||
"""登录 sandbox 平台 → 抓合同表 + 材料 → 落库。
|
||
|
||
human_wait_timeout: 手工登录 fallback 等待秒数（默认 HUMAN_WAIT_TIMEOUT=180）。
|
||
失败时可能抛出 RPAError（LoginTimeoutError / MissingBrowserError / TableNotFoundError）。
|
||
"""
|
||
# Lazy import: 读 facade 持有的 _safe_getenv_dir / 异常类 / PLATFORM_URL 等
|
||
from service import platform_collector as _pc
|
||
|
||
try:
|
||
from playwright.sync_api import sync_playwright
|
||
except ImportError:
|
||
logger.error("playwright 未安装，跳过 RPA 采集")
|
||
return _error_result("playwright_not_installed", 0)
|
||
|
||
profile_dir = _pc._safe_getenv_dir("SMARTCAMPUS_PROFILE_DIR", "playwright-profile")
|
||
download_dir = _pc._safe_getenv_dir("MATERIALS_DOWNLOAD_DIR", "materials")
|
||
profile_dir.mkdir(parents=True, exist_ok=True)
|
||
download_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
logger.info(
|
||
"platform_collector: 启动 profile=%s download_dir=%s headless=%s limit=%d "
|
||
"human_wait_timeout=%d",
|
||
profile_dir, download_dir, headless, limit, human_wait_timeout,
|
||
)
|
||
|
||
contracts: list[dict] = []
|
||
fetched_rows = 0
|
||
downloaded = 0
|
||
|
||
with sync_playwright() as p:
|
||
# v1.1.9：headed 模式优先用系统 Edge（跟 Windows 桌面 session 同源，
|
||
# 能真正弹窗让你看到并手工登录）；找不到则 fallback Playwright 内置 chromium
|
||
if not headless:
|
||
try:
|
||
ctx = p.chromium.launch_persistent_context(
|
||
user_data_dir=str(profile_dir),
|
||
headless=False,
|
||
accept_downloads=True,
|
||
channel="msedge",
|
||
viewport={"width": 1440, "height": 900},
|
||
)
|
||
logger.info(
|
||
"platform_collector: 使用系统 Edge（headed 弹窗）"
|
||
)
|
||
except Exception as e:
|
||
logger.warning(
|
||
"platform_collector: 系统 Edge 启动失败（%s），fallback Playwright chromium",
|
||
e,
|
||
)
|
||
ctx = p.chromium.launch_persistent_context(
|
||
user_data_dir=str(profile_dir),
|
||
headless=False,
|
||
accept_downloads=True,
|
||
viewport={"width": 1440, "height": 900},
|
||
)
|
||
else:
|
||
ctx = p.chromium.launch_persistent_context(
|
||
user_data_dir=str(profile_dir),
|
||
headless=True,
|
||
accept_downloads=True,
|
||
viewport={"width": 1440, "height": 900},
|
||
)
|
||
try:
|
||
page = ctx.new_page()
|
||
# v1.1.6：覆盖 Playwright 默认操作超时（之前无 set_default_timeout，导致
|
||
# 所有 wait_for_selector 默认 30s，调试时不直观）
|
||
page.set_default_timeout(DEFAULT_PAGE_TIMEOUT_MS)
|
||
|
||
# 1. v1.1.21：先 goto **根域**（sandbox 登录页），让 SPA hydrate 2 秒
|
||
logger.info(
|
||
"platform_collector: 启动，goto ROOT_URL=%s (当前 URL=%s)",
|
||
_pc.ROOT_URL, page.url,
|
||
)
|
||
_pc._goto(page, _pc.ROOT_URL)
|
||
# 给 React/Vue 框架 2 秒 hydrate 时间，看登录 input 是否就绪
|
||
page.wait_for_timeout(2_000)
|
||
|
||
if _pc._is_logged_in(page):
|
||
logger.info(
|
||
"platform_collector: ✅ 已登录态（URL=%s），跳过登录流程",
|
||
page.url,
|
||
)
|
||
else:
|
||
# 2. 等 #jc-user / #jc-pass 出现（最多 2 秒）
|
||
# 若 sandbox 已预填 demo/123456 → 自动点 Sign in
|
||
# 若 input 为空 → 等用户输入（**v1.1.24 用户自己点登录按钮**）
|
||
auto_clicked = _pc._try_auto_sign_in(page, wait_input_sec=2)
|
||
if auto_clicked:
|
||
logger.info(
|
||
"platform_collector: ✅ 自动登录成功（预填 demo），等 URL 跳到 /app/"
|
||
)
|
||
elif not headless:
|
||
# 3. input 为空：等用户输入 → **用户自己点 Sign in**（脚本不抢按钮）
|
||
if _wait_for_login_inputs_filled(page, wait_sec=human_wait_timeout):
|
||
logger.info(
|
||
"platform_collector: ✅ 用户完成输入并跳转，等 URL 跳到 /app/"
|
||
)
|
||
else:
|
||
# 超时未跳转 → fallback 到原来的多信号等待
|
||
logger.warning(
|
||
"platform_collector: 用户未跳转，fallback 到 _wait_for_manual_login"
|
||
)
|
||
_wait_for_manual_login(page, wait_sec=human_wait_timeout)
|
||
else:
|
||
raise _pc.LoginTimeoutError(
|
||
f"headless 模式下未登录，且无法手工登录"
|
||
f"（请用 --show-browser 重试）"
|
||
)
|
||
|
||
# 4. v1.1.27：等 URL 跳到任一「SPA 已路由」状态（最多 30s，**不做任何操作**）
|
||
# - /app/ = 已到 SPA 主页
|
||
# - /select = sandbox 中间页
|
||
# - /login 或 /signin = sandbox 又把人踢回登录页（根域登录后常见，
|
||
# sandbox 会跳到 /industries/campus/smartcampus/login SPA 二级登录页）
|
||
# - 如果 URL 跳到 login/signin，**立即**退出 30 秒等待（不再干等），
|
||
# 让 4.5 处理。否则会等满 30 秒才发现要进 SPA 登录兜底，造成卡顿。
|
||
spa_url_seen = False
|
||
if not _wait_for_url_stable(
|
||
page, ("/app/", "/select", "/login", "/signin"), max_wait_sec=30,
|
||
):
|
||
logger.warning(
|
||
"platform_collector: 登录后 30s 内 URL 未稳定到 /app/、/select、"
|
||
"/login 或 /signin（当前 URL=%s）",
|
||
page.url,
|
||
)
|
||
|
||
# 4.5. v1.1.28/31：URL 落在 SPA 二级登录页 → 主动等 input hydrate（最多 10 秒）
|
||
# → 从 .env 读凭证 fill（v1.1.31：密码不硬编码）→ 点登录 → 等 URL 跳离
|
||
# v1.1.27 失败的根因：URL 已稳定在 /login 时，React SPA input 可能还没渲染，
|
||
# 找不到 input → False → 卡 180 秒等用户输入。
|
||
# 修复：先等 input 真正 hydrate 出来，再 fill+click。
|
||
cur_url_after_stable = page.url.lower()
|
||
if "/login" in cur_url_after_stable or "/signin" in cur_url_after_stable:
|
||
logger.warning(
|
||
"platform_collector: 登录后落在 SPA 二级登录页（%s），"
|
||
"v1.1.31 触发 SPA 登录兜底（等 input hydrate + 从 .env 读凭证 fill）",
|
||
page.url,
|
||
)
|
||
# 1. 等 input 真正 hydrate 出来（最多 10 秒）
|
||
try:
|
||
page.wait_for_selector(
|
||
"#jc-user, #jc-pass, input[type='tel'], input[type='password']",
|
||
timeout=10_000,
|
||
state="attached",
|
||
)
|
||
page.wait_for_timeout(500) # 让 React 状态稳定
|
||
except Exception as e:
|
||
logger.warning(
|
||
"platform_collector: SPA 登录页 10s 内未渲染 input: %s", e,
|
||
)
|
||
|
||
# 2. 从 .env 读凭证 fill + 点登录
|
||
spa_filled = _fill_spa_login_from_env(page)
|
||
if spa_filled:
|
||
logger.info(
|
||
"platform_collector: ✅ SPA 二级登录页（来自 .env）已自动填好并点登录，"
|
||
"等 URL 跳到 /app/ 或 /select"
|
||
)
|
||
# 等 URL 跳离 login（30 秒内跳到 /app/ 或 /select 都算成功）
|
||
if not _wait_for_url_stable(page, ("/app/", "/select"), max_wait_sec=30):
|
||
logger.warning(
|
||
"platform_collector: SPA 登录后 30s 内 URL 未稳定到 /app/或/select"
|
||
"（当前 URL=%s，.env 凭证可能失效）",
|
||
page.url,
|
||
)
|
||
# 凭证失效：等用户手工登录（headless 模式直接抛错）
|
||
if not headless:
|
||
_wait_for_manual_login(page, wait_sec=human_wait_timeout)
|
||
else:
|
||
raise _pc.LoginTimeoutError(
|
||
f"SPA 二级登录页 .env 凭证 fill 后 URL 不跳离"
|
||
f"（可能凭证失效，URL={page.url}）"
|
||
)
|
||
else:
|
||
# .env 无凭证 / input 仍未渲染 → 等用户手工输入
|
||
logger.warning(
|
||
"platform_collector: SPA 登录页自动 fill 失败"
|
||
"（.env 无凭证或 input 未渲染），fallback 等用户输入"
|
||
)
|
||
if not headless:
|
||
if _wait_for_login_inputs_filled(page, wait_sec=human_wait_timeout):
|
||
_wait_for_url_stable(page, ("/app/", "/select"), max_wait_sec=30)
|
||
else:
|
||
_wait_for_manual_login(page, wait_sec=human_wait_timeout)
|
||
else:
|
||
raise _pc.LoginTimeoutError(
|
||
f"SPA 二级登录页 .env 无凭证且 headless 无法手工登录"
|
||
f"（URL={page.url}）"
|
||
)
|
||
|
||
# 5. v1.1.22：显式跳到 SPA_HOME_URL（/app/home），等侧边栏 link 出现
|
||
# 避免 sandbox 在 /select 中间页时直接找 sidebar link 找不到
|
||
# v1.1.32 优化：URL 不在 /app/home（可能在 /select 中间页）就 goto（5s 等）
|
||
if "/app/home" not in page.url.lower():
|
||
logger.info(
|
||
"platform_collector: 当前 URL=%s，跳到 SPA_HOME_URL=%s",
|
||
page.url, _pc.SPA_HOME_URL,
|
||
)
|
||
_pc._goto(page, _pc.SPA_HOME_URL)
|
||
page.wait_for_timeout(SPA_HYDRATION_MS)
|
||
# v1.1.32 优化：等 URL 稳定到 /app/home（最多 5s，比 15s 快很多）
|
||
if not _wait_for_url_stable(page, "/app/home", max_wait_sec=5):
|
||
logger.warning(
|
||
"platform_collector: 5s 内未稳定到 /app/home（当前 URL=%s）",
|
||
page.url,
|
||
)
|
||
|
||
# 6. 等 SPA 渲染 + click 侧边栏 hr/contracts（SPA 路由，避免 goto 触发整页 reload 丢 session）
|
||
try:
|
||
# dump SPA home 一次（用于调试 selector）
|
||
page.wait_for_timeout(POST_LOGIN_SETTLE_MS)
|
||
_pc._dump_page(page, download_dir.parent / "spa_home_dump.html", "SPA home")
|
||
|
||
# 等 SPA 真正加载完（侧边栏 link 出现，最长 SIDEBAR_LINK_TIMEOUT_MS）
|
||
try:
|
||
page.wait_for_selector(
|
||
"a[href$='/hr/contracts']",
|
||
timeout=SIDEBAR_LINK_TIMEOUT_MS,
|
||
state="attached",
|
||
)
|
||
except Exception as e:
|
||
logger.warning("等侧边栏 hr/contracts link 超时: %s", e)
|
||
|
||
link = page.locator("a[href$='/hr/contracts']").first
|
||
if link.count() > 0:
|
||
link.click()
|
||
logger.info("platform_collector: click 侧边栏 hr/contracts")
|
||
try:
|
||
page.wait_for_url(
|
||
lambda u: "/hr/contracts" in u,
|
||
timeout=SIDEBAR_CLICK_URL_TIMEOUT_MS,
|
||
)
|
||
except Exception as e:
|
||
logger.warning("等 /hr/contracts URL 超时: %s；当前 URL=%s",
|
||
e, page.url)
|
||
else:
|
||
logger.info(
|
||
"platform_collector: 侧边栏 link 仍未出现，"
|
||
"使用 SPA 路由（pushState）到 hr/contracts, 当前 URL=%s",
|
||
page.url,
|
||
)
|
||
# 用 SPA 内部路由而非整页 goto，避免 session 被 server 端踢回 login
|
||
try:
|
||
page.evaluate(
|
||
"u => history.pushState({}, '', u) || "
|
||
"window.dispatchEvent(new PopStateEvent('popstate')),"
|
||
" '/industries/campus/smartcampus/app/hr/contracts'"
|
||
)
|
||
page.wait_for_timeout(PUSHSTATE_SETTLE_MS)
|
||
# v1.1.32：pushState 后 URL 可能没切（sandbox mock 偶发），
|
||
# 检查 URL 真到 /hr/contracts 才算成功，否则 fallback goto
|
||
if "/hr/contracts" not in page.url.lower():
|
||
logger.warning(
|
||
"platform_collector: pushState 后 URL=%s 未到 /hr/contracts，"
|
||
"fallback goto PLATFORM_URL",
|
||
page.url,
|
||
)
|
||
_pc._goto(page, _pc.PLATFORM_URL)
|
||
except Exception as e:
|
||
logger.warning("SPA pushState 失败: %s", e)
|
||
_pc._goto(page, _pc.PLATFORM_URL)
|
||
except Exception as e:
|
||
logger.warning("click 侧边栏失败: %s；fallback goto", e)
|
||
_pc._goto(page, _pc.PLATFORM_URL)
|
||
|
||
logger.info("platform_collector: 当前 URL=%s", page.url)
|
||
|
||
# 5.5. v1.1.8：**首次登录后不再强制刷新**，避免打断用户手工登录态。
|
||
# 改为「等表格出现」兜底：给 SPA 一个初始等待窗口（TABLE_INITIAL_WAIT_MS*2），
|
||
# 如果表格仍未出现，**才**刷新一次（sandbox mock 异步加载场景）。
|
||
logger.info(
|
||
"platform_collector: 进入 hr/contracts 后**不刷新**，等待表格出现"
|
||
"（首次登录态不要打断）"
|
||
)
|
||
|
||
# 5.6. 兜底刷新：仅当等表格超时后才触发；刷新后可能被踢回 login，需重新登录
|
||
# 修复：之前没检查 URL，刷新后停在 login 页继续往后走，找不到表格就 dump 报错
|
||
# v1.1.32 优化：刷回 login 时用 _fill_spa_login_from_env（从 .env 读），
|
||
# 不再调硬编码 demo/123456 的 _fill_spa_login（与 .env 凭证一致）。
|
||
def _try_reload_after_table_timeout():
|
||
logger.warning(
|
||
"platform_collector: 表格未在 %dms 内出现，触发兜底刷新",
|
||
TABLE_INITIAL_WAIT_MS * 2,
|
||
)
|
||
try:
|
||
page.reload(
|
||
wait_until="networkidle",
|
||
timeout=RELOAD_NETWORKIDLE_TIMEOUT_MS,
|
||
)
|
||
except Exception as e:
|
||
logger.warning("刷新 networkidle 失败: %s；尝试 domcontentloaded", e)
|
||
try:
|
||
page.reload(
|
||
wait_until="domcontentloaded",
|
||
timeout=RELOAD_DOMCONTENTLOADED_TIMEOUT_MS,
|
||
)
|
||
except Exception as e2:
|
||
logger.warning("刷新 domcontentloaded 也失败: %s", e2)
|
||
if "/login" in page.url.lower() or "/signin" in page.url.lower():
|
||
logger.warning(
|
||
"platform_collector: 刷新后回到登录页（%s），重新登录",
|
||
page.url,
|
||
)
|
||
# v1.1.32：先等 input hydrate，再走 .env 凭证 fill
|
||
try:
|
||
page.wait_for_selector(
|
||
"#jc-user, #jc-pass, input[type='tel'], input[type='password']",
|
||
timeout=10_000,
|
||
state="attached",
|
||
)
|
||
page.wait_for_timeout(500)
|
||
except Exception as e:
|
||
logger.warning(
|
||
"platform_collector: 刷新后 login 页 10s 内未渲染 input: %s", e,
|
||
)
|
||
if not _fill_spa_login_from_env(page):
|
||
# v1.1.32：.env 无凭证时 fallback 到硬编码 demo（保持 v1.1.10 兼容）
|
||
logger.warning(
|
||
"platform_collector: .env 无凭证，fallback 硬编码 demo/123456"
|
||
)
|
||
if not _fill_spa_login(page):
|
||
if not headless:
|
||
_wait_for_manual_login(page, wait_sec=human_wait_timeout)
|
||
else:
|
||
raise _pc.LoginTimeoutError(
|
||
"刷新后回到 login 页且 headless 无法手工登录"
|
||
)
|
||
else:
|
||
page.wait_for_timeout(POST_LOGIN_SETTLE_MS)
|
||
else:
|
||
# .env 凭证 fill 成功 → 等 URL 跳离 login
|
||
_wait_for_url_stable(page, ("/app/", "/select"), max_wait_sec=15)
|
||
_pc._goto(page, _pc.PLATFORM_URL)
|
||
page.wait_for_timeout(POST_RELOAD_LOGIN_HYDRATION_MS)
|
||
|
||
# 7. dump 当前 hr/contracts 页面（用于调试）
|
||
_pc._dump_page(page, download_dir.parent / "hr_contracts_dump.html", "hr/contracts")
|
||
|
||
# 8. 等待表格加载（v1.1.8：先等，**表格超时未出现才触发兜底刷新**——
|
||
# 修复「首次登录后强制刷新打断手工登录态」问题）
|
||
page.wait_for_timeout(TABLE_INITIAL_WAIT_MS) # 给 React 初始化 + mock 数据填充时间
|
||
found_table = False
|
||
for attempt in range(TABLE_POLL_MAX_ATTEMPTS):
|
||
page.wait_for_timeout(TABLE_POLL_INTERVAL_MS)
|
||
cur_url = page.url
|
||
try:
|
||
page.wait_for_selector(
|
||
"table tbody tr",
|
||
timeout=TABLE_TR_TIMEOUT_MS,
|
||
)
|
||
found_table = True
|
||
break
|
||
except Exception:
|
||
logger.info(
|
||
"platform_collector: URL=%s 等待表格加载 (attempt %d/%d)...",
|
||
cur_url, attempt + 1, TABLE_POLL_MAX_ATTEMPTS,
|
||
)
|
||
|
||
if not found_table:
|
||
# v1.1.8：表格没出现，**先尝试兜底刷新一次**（sandbox mock 异步加载场景）；
|
||
# 刷新后再次轮询表格，仍找不到才报 TableNotFoundError。
|
||
logger.warning(
|
||
"platform_collector: 首次轮询未找到表格，触发兜底刷新（sandbox mock 异步加载场景）"
|
||
)
|
||
_try_reload_after_table_timeout()
|
||
# 刷新后重新轮询一次（最多 5 次）
|
||
for attempt2 in range(5):
|
||
page.wait_for_timeout(TABLE_POLL_INTERVAL_MS)
|
||
try:
|
||
page.wait_for_selector(
|
||
"table tbody tr",
|
||
timeout=TABLE_TR_TIMEOUT_MS,
|
||
)
|
||
found_table = True
|
||
logger.info(
|
||
"platform_collector: 兜底刷新后找到表格 (attempt %d)", attempt2 + 1
|
||
)
|
||
break
|
||
except Exception:
|
||
logger.info(
|
||
"platform_collector: 兜底刷新后仍等待表格 (attempt %d/5)...",
|
||
attempt2 + 1,
|
||
)
|
||
|
||
if not found_table:
|
||
_pc._dump_page(page, download_dir.parent / "sandbox_dump.html", "sandbox")
|
||
logger.warning("platform_collector: 表格未找到（兜底刷新后仍未出现）")
|
||
raise _pc.TableNotFoundError(
|
||
f"{TABLE_POLL_MAX_ATTEMPTS * TABLE_POLL_INTERVAL_MS // 1000}s "
|
||
f"内未找到表格（兜底刷新后仍未出现） (URL={page.url})"
|
||
)
|
||
|
||
# 表格已找到，等 tbody 完全填充 + dump 一份完整 DOM（用于诊断按钮 selector）
|
||
try:
|
||
page.wait_for_function(
|
||
"document.querySelectorAll('table tbody tr').length > 0",
|
||
timeout=DEFAULT_PAGE_TIMEOUT_MS,
|
||
)
|
||
page.wait_for_timeout(FULL_RENDER_WAIT_MS)
|
||
_pc._dump_page(
|
||
page,
|
||
download_dir.parent / "hr_contracts_full_dump.html",
|
||
"完整 hr/contracts",
|
||
)
|
||
except Exception as e:
|
||
logger.warning("等完整渲染超时: %s", e)
|
||
|
||
rows = page.eval_on_selector_all(
|
||
"table tbody tr",
|
||
"rs => rs.map(r => Array.from(r.querySelectorAll('td')).map(td => td.innerText))",
|
||
)
|
||
fetched_rows = len(rows)
|
||
logger.info("platform_collector: 抓到 %d 行", fetched_rows)
|
||
|
||
# 检查每行是否有「续聘材料包」按钮
|
||
try:
|
||
mat_btn_count = page.locator(
|
||
"table tbody tr button:has-text('续聘材料包')"
|
||
).count()
|
||
logger.info(
|
||
"platform_collector: 「续聘材料包」按钮数=%d", mat_btn_count,
|
||
)
|
||
except Exception:
|
||
mat_btn_count = 0
|
||
|
||
# 抓「材料包」card 区段的 DOM（用于后续解析每行 click 后的卡片文本）
|
||
# sandbox 行为：click 按钮后会在表格下方追加「续聘准备任务书」+「单位清单」卡片
|
||
|
||
for i, r in enumerate(rows):
|
||
if limit and i >= limit:
|
||
break
|
||
# v1.1.8：表格列数自适应 —— sandbox 首次登录未刷新是 7 列
|
||
# （姓名/单位/到期日/剩余天数/预警/建议/按钮），刷新后才是 10 列
|
||
# （多 工号/岗位/合同类型/起止日期）。两种都支持。
|
||
ncols = len(r)
|
||
if ncols < 6:
|
||
logger.warning(
|
||
"行 %d 列数=%d < 6，跳过: %s", i, ncols, r
|
||
)
|
||
continue
|
||
|
||
# 列序探测：判断首列是不是工号（纯数字 + 字母，纯姓名是中文）
|
||
first_col = r[0].strip()
|
||
import re as _re
|
||
is_emp_id = bool(_re.match(r"^[A-Za-z]?\d+$", first_col))
|
||
|
||
if is_emp_id and ncols >= 9:
|
||
# 10 列布局（含工号）：0 工号 1 姓名 2 单位 3 岗位 4 合同类型
|
||
# 5 起止(含~) 6 剩余天数 7 预警 8 建议 9 按钮
|
||
emp_id = first_col
|
||
teacher = r[1].strip()
|
||
unit = r[2].strip()
|
||
period = r[5].strip()
|
||
contract_end = period.split("~")[-1].strip() if "~" in period else period
|
||
days_raw = r[6].strip()
|
||
alert_label = r[7].strip()
|
||
suggestion_hint = r[8].strip()
|
||
elif ncols >= 6:
|
||
# 7 列布局（无工号）：0 姓名 1 单位 2 到期日 3 剩余天数 4 预警 5 建议 6 按钮
|
||
teacher = first_col
|
||
unit = r[1].strip()
|
||
contract_end = r[2].strip()
|
||
days_raw = r[3].strip()
|
||
alert_label = r[4].strip()
|
||
suggestion_hint = r[5].strip()
|
||
emp_id = "" # 工号缺失，留空让 _gen_contract_id 用 teacher_name 兜底
|
||
else:
|
||
logger.warning(
|
||
"行 %d 列数=%d 无法识别布局，跳过: %s", i, ncols, r
|
||
)
|
||
continue
|
||
|
||
# 用工号生成 contract_id（稳定，不会随姓名变化）
|
||
# v1.1.8：7 列布局无工号 → 用 teacher_name 派生（避免与 10 列布局冲突）
|
||
if emp_id:
|
||
contract_id = _pc._gen_contract_id(emp_id, contract_end)
|
||
else:
|
||
contract_id = _pc._gen_contract_id(teacher, contract_end)
|
||
contract = {
|
||
"contract_id": contract_id,
|
||
"teacher_name": teacher,
|
||
"unit_name": unit,
|
||
"contract_end": _pc._normalize_date(contract_end),
|
||
"days_to_expire_hint": _pc._parse_days(days_raw),
|
||
"alert_level_hint": _pc._parse_alert(alert_label),
|
||
"ai_suggestion_hint": suggestion_hint,
|
||
"materials": [],
|
||
}
|
||
|
||
# 点「续聘材料包」按钮 → 等卡片展开 → 抓卡片文本（含 checklist）
|
||
if mat_btn_count > 0:
|
||
try:
|
||
btn = page.locator("table tbody tr").nth(i).locator(
|
||
"button:has-text('续聘材料包')"
|
||
)
|
||
btn.click()
|
||
# 等卡片出现（页面下方追加 h2「续聘准备任务书」/「单位清单」）
|
||
page.wait_for_timeout(CARD_APPEAR_WAIT_MS)
|
||
# 抓表格下方所有 card 文本
|
||
cards_text = page.locator(
|
||
"main .grid > div, main > div > div > div"
|
||
).all_inner_texts()
|
||
materials_text = "\n\n".join(
|
||
t.strip() for t in cards_text if t.strip()
|
||
)
|
||
if materials_text:
|
||
# 存到 materials_content（sandbox mock 卡片文本作为材料包内容）
|
||
contract["materials"].append({
|
||
"kind": "sandbox_card",
|
||
"file_name": f"{contract_id}_materials.txt",
|
||
"text": materials_text,
|
||
})
|
||
downloaded += 1
|
||
logger.info(
|
||
"platform_collector: 抓到卡片文本 %d 字 (%s)",
|
||
len(materials_text), contract_id,
|
||
)
|
||
# dump 一次（i==0）方便调试
|
||
if i == 0:
|
||
_pc._dump_page(
|
||
page,
|
||
download_dir.parent / "after_click_dump.html",
|
||
"首行 click 后",
|
||
)
|
||
except Exception as e:
|
||
logger.warning("抓卡片失败 %s: %s", contract_id, e)
|
||
else:
|
||
if i == 0:
|
||
logger.info(
|
||
"platform_collector: sandbox 未部署「续聘材料包」按钮，"
|
||
"跳过材料抓取（仅采集表格列）",
|
||
)
|
||
|
||
contracts.append(contract)
|
||
finally:
|
||
ctx.close()
|
||
|
||
saved = _pc._persist_contracts(contracts)
|
||
logger.info(
|
||
"platform_collector: contracts 落库完成 saved=%d / total_fetched=%d",
|
||
saved, len(contracts),
|
||
)
|
||
|
||
# v1.1.18：每个合同入队 download_tasks（独立事务，失败不影响主流程）
|
||
contract_ids = [c["contract_id"] for c in contracts if c.get("contract_id")]
|
||
enqueue_summary = _pc._enqueue_download_tasks_safe(contract_ids)
|
||
|
||
return {
|
||
"platform_url": _pc.PLATFORM_URL,
|
||
"fetched_rows": fetched_rows,
|
||
"saved_contracts": saved,
|
||
"downloaded_materials": downloaded,
|
||
"download_queue": enqueue_summary,
|
||
"contracts": contracts,
|
||
}
|
||
|
||
|
||
def _error_result(reason: str, fetched: int) -> dict:
|
||
# Lazy import: 读 facade 的 PLATFORM_URL
|
||
from service import platform_collector as _pc
|
||
|
||
return {
|
||
"platform_url": _pc.PLATFORM_URL,
|
||
"fetched_rows": fetched,
|
||
"saved_contracts": 0,
|
||
"downloaded_materials": 0,
|
||
"contracts": [],
|
||
"error": reason,
|
||
}
|