taojianmin
6437d6e414
All checks were successful
技能自动化发布 / release (push) Successful in 7s
按 skill-template examples/real_browser_rpa 设计模式重构 platform_collector，
把 v1.1.5 修复的硬编码等待时间统一抽到 constants.py，并补上手工登录 fallback
和结构化错误码。
核心改动：
- constants.py 新增 24 个 RPA 常量（HUMAN_WAIT_TIMEOUT=180 / HUMAN_POLL_INTERVAL=2.0 /
DEFAULT_PAGE_TIMEOUT_MS=15s / GOTO_TIMEOUT_MS=60s / SPA_LOGIN_* / TABLE_* /
SIDEBAR_LINK_TIMEOUT_MS 等），与 skill-template human_verification 同名常量对齐
- platform_collector 重构：
* 新增 RPAError / LoginTimeoutError (ERROR:LOGIN_TIMEOUT) /
MissingBrowserError (ERROR:MISSING_BROWSER) / TableNotFoundError (ERROR:TABLE_NOT_FOUND)
* 新增 _is_logged_in / _wait_for_manual_login（轮询 + 180s 兜底，与
skill-template task_rpa.py::_ensure_logged_in 对齐）
* _fill_spa_login 自动 fill 失败时 → 自动 fallback _wait_for_manual_login
（仅 --show-browser 模式；headless 模式抛 LoginTimeoutError 退出码 2）
* 所有 wait_for_timeout / wait_for_selector timeout 替换为常量
* collect_all() 加 human_wait_timeout 参数（CLI 透传）
* page.set_default_timeout(DEFAULT_PAGE_TIMEOUT_MS) 覆盖 Playwright 默认 15s
- app.py cmd_collect_platform：
* 捕获 RPAError → JSON 输出 {error: e.code, detail: ...} + exit 2
* 新增 --human-wait-timeout N flag
- requirements.txt：删除违规的 playwright>=1.40.0（playwright 由宿主 jiangchang_skill_core
runtime 提供，文件顶部注释已声明）
- .env.example：新增 HUMAN_WAIT_TIMEOUT / RPA_PAGE_TIMEOUT_MS / RPA_GOTO_TIMEOUT_MS
覆盖项
- tests/run_tests.py：新增 7 个 platform_collector 纯函数测试
（_gen_contract_id × 2 / _normalize_date × 2 / _parse_days × 2 / _parse_alert）
- SKILL.md version 1.1.5 → 1.1.6；CHANGELOG.md 加 1.1.6 条目
测试：28/28 通过（21 原有 + 7 新增）
POLICY-RELEASE-001：SKILL.md=1.1.6 == CHANGELOG=1.1.6 == SKILL_VERSION=1.1.6
Co-Authored-By: Claude Code <noreply@anthropic.com>
761 lines
31 KiB
Python
761 lines
31 KiB
Python
"""Playwright RPA：从 SmartCampus HR 平台抓合同表 + 续聘材料包。
|
||
|
||
设计要点（v1.1.6，按 skill-template examples/real_browser_rpa 设计模式）：
|
||
- 使用 launch_persistent_context 复用浏览器登录态，避免密码硬编码
|
||
- 首次使用需用户手工登录一次（CLI 默认 --headless false 便于交互）
|
||
- 后续使用无头模式（profile 已存 cookie）
|
||
- **登录策略**：自动 fill demo/123456（sandbox 已知凭据） + 失败时 fallback 手工登录等待（轮询 180s）
|
||
- 抓表格：DOM eval_on_selector_all → JSON
|
||
- 抓材料：点 "续聘材料包" 按钮 → 抓卡片 inner_text
|
||
|
||
依赖为可选：playwright 未安装时 import 抛错，上层 CLI 捕获并降级报错。
|
||
|
||
导航流程（v1.1.1 强化版 — 强制 goto）：
|
||
1. goto 根域
|
||
2. 根域 login（如仍显示表单）→ 等跳到 /home 或 /select
|
||
3. 强制 goto sandbox SPA login URL
|
||
4. SPA login：自动 fill demo/123456，失败 fallback 手工等待（HUMAN_WAIT_TIMEOUT=180s）
|
||
5. 强制 goto SPA_HOME
|
||
6. click 侧边栏 hr/contracts
|
||
7. 等 table tbody tr（最长 ~40s）
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import re
|
||
import sqlite3
|
||
import time
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
|
||
from db import repository as repo
|
||
from util.constants import (
|
||
CARD_APPEAR_WAIT_MS,
|
||
DEFAULT_PAGE_TIMEOUT_MS,
|
||
FULL_RENDER_WAIT_MS,
|
||
GOTO_TIMEOUT_MS,
|
||
HUMAN_POLL_INTERVAL,
|
||
HUMAN_WAIT_TIMEOUT,
|
||
POST_LOGIN_SETTLE_MS,
|
||
POST_RELOAD_LOGIN_HYDRATION_MS,
|
||
POST_RELOGIN_GOTO_WAIT_MS,
|
||
PUSHSTATE_SETTLE_MS,
|
||
RELOAD_DOMCONTENTLOADED_TIMEOUT_MS,
|
||
RELOAD_NETWORKIDLE_TIMEOUT_MS,
|
||
SIDEBAR_CLICK_URL_TIMEOUT_MS,
|
||
SIDEBAR_LINK_TIMEOUT_MS,
|
||
SPA_HOME_RETRY_DELAY_MS,
|
||
SPA_HOME_RETRY_MAX,
|
||
SPA_HYDRATION_MS,
|
||
SPA_LOGIN_INPUT_TIMEOUT_MS,
|
||
SPA_LOGIN_SETTLE_MS,
|
||
SPA_LOGIN_URL_TIMEOUT_MS,
|
||
TABLE_INITIAL_WAIT_MS,
|
||
TABLE_POLL_INTERVAL_MS,
|
||
TABLE_POLL_MAX_ATTEMPTS,
|
||
TABLE_TR_TIMEOUT_MS,
|
||
URL_AFTER_LOGIN_TIMEOUT_MS,
|
||
)
|
||
from util.logging_setup import get_skill_logger
|
||
from util.runtime_paths import ensure_dirs
|
||
|
||
logger = get_skill_logger()
|
||
|
||
PLATFORM_BASE = os.environ.get("SMARTCAMPUS_BASE_URL", "https://sandbox.jc2009.com")
|
||
PLATFORM_HR_PATH = os.environ.get(
|
||
"SMARTCAMPUS_HR_PATH", "/industries/campus/smartcampus/app/hr/contracts"
|
||
)
|
||
PLATFORM_URL = PLATFORM_BASE + PLATFORM_HR_PATH
|
||
|
||
ROOT_URL = PLATFORM_BASE + "/"
|
||
SELECT_URL = PLATFORM_BASE + "/select"
|
||
SPA_HOME_URL = PLATFORM_BASE + "/industries/campus/smartcampus/app/home"
|
||
SPA_LOGIN_URL = PLATFORM_BASE + "/industries/campus/smartcampus/login"
|
||
|
||
|
||
# ---------- 错误码（与 skill-template 对齐） ----------
|
||
|
||
class RPAError(Exception):
|
||
"""RPA 流程错误基类。"""
|
||
|
||
code: str = "RPA_ERROR"
|
||
|
||
def __init__(self, message: str):
|
||
super().__init__(message)
|
||
self.message = message
|
||
|
||
|
||
class LoginTimeoutError(RPAError):
|
||
"""手工登录超时（HUMAN_WAIT_TIMEOUT 秒内未检测到登录成功）。
|
||
|
||
错误码：ERROR:LOGIN_TIMEOUT（与 skill-template task_rpa.py 对齐）
|
||
"""
|
||
|
||
code = "ERROR:LOGIN_TIMEOUT"
|
||
|
||
|
||
class MissingBrowserError(RPAError):
|
||
"""未检测到 Chrome/Edge。
|
||
|
||
错误码：ERROR:MISSING_BROWSER（与 skill-template browser_session.py 对齐）
|
||
"""
|
||
|
||
code = "ERROR:MISSING_BROWSER"
|
||
|
||
|
||
class TableNotFoundError(RPAError):
|
||
"""40s 内未找到表格。
|
||
|
||
错误码：ERROR:TABLE_NOT_FOUND
|
||
"""
|
||
|
||
code = "ERROR:TABLE_NOT_FOUND"
|
||
|
||
|
||
# ---------- 工具函数 ----------
|
||
|
||
def _safe_getenv_dir(env_key: str, default_rel: str) -> Path:
|
||
raw = os.environ.get(env_key)
|
||
if raw:
|
||
return Path(raw).resolve()
|
||
from util.runtime_paths import get_data_dir
|
||
|
||
return (get_data_dir() / default_rel).resolve()
|
||
|
||
|
||
def _goto(page, url: str, timeout: int = GOTO_TIMEOUT_MS) -> None:
|
||
"""带 networkidle / domcontentloaded 双兜底的 goto。"""
|
||
try:
|
||
page.goto(url, wait_until="networkidle", timeout=timeout)
|
||
except Exception as e:
|
||
logger.warning("goto networkidle 失败 %s: %s；尝试 domcontentloaded", url, e)
|
||
try:
|
||
page.goto(url, wait_until="domcontentloaded", timeout=timeout)
|
||
except Exception as e2:
|
||
logger.warning("goto domcontentloaded 也失败 %s: %s", url, e2)
|
||
|
||
|
||
def _dump_page(page, path: Path, label: str) -> None:
|
||
"""Dump 当前页面 HTML 到指定路径，自动创建父目录。修复：父目录可能不存在导致静默失败。"""
|
||
try:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
html = page.content()
|
||
path.write_text(html, encoding="utf-8")
|
||
logger.info(
|
||
"platform_collector: %s dump → %s (%d 字节)",
|
||
label, path, len(html),
|
||
)
|
||
except Exception as e:
|
||
logger.warning("dump %s 失败: %s", label, e)
|
||
|
||
|
||
def _is_logged_in(page) -> bool:
|
||
"""检测 sandbox SPA 是否已登录（URL 含 /app/ 且不含 /login 或 /signin）。
|
||
|
||
与 skill-template real_browser_rpa _is_logged_in 概念对齐（这里只看 URL，
|
||
因为 sandbox SPA 登录后 URL 稳定跳到 /app/...）。
|
||
"""
|
||
url_lower = page.url.lower()
|
||
return (
|
||
"/app/" in url_lower
|
||
and "/login" not in url_lower
|
||
and "/signin" not in url_lower
|
||
)
|
||
|
||
|
||
def _wait_for_manual_login(page, wait_sec: int = HUMAN_WAIT_TIMEOUT) -> bool:
|
||
"""手工登录 fallback：轮询 _is_logged_in，每 HUMAN_POLL_INTERVAL 秒一次。
|
||
|
||
与 skill-template task_rpa.py::_ensure_logged_in 设计对齐：
|
||
- 不自动破解，等用户在浏览器手工完成登录
|
||
- 屏幕打印 [登录] 提示
|
||
- 超时抛 LoginTimeoutError（ERROR:LOGIN_TIMEOUT）
|
||
|
||
自动 fill demo/123456 失败时调用此函数。
|
||
"""
|
||
print(
|
||
f"[登录] 自动登录失败，请在浏览器中完成登录"
|
||
f"（sandbox demo/123456 或真实账号），最多等待 {wait_sec} 秒..."
|
||
)
|
||
deadline = time.monotonic() + wait_sec
|
||
attempt = 0
|
||
while time.monotonic() < deadline:
|
||
attempt += 1
|
||
if _is_logged_in(page):
|
||
print(f"[登录] 检测到登录成功（attempt {attempt}）")
|
||
return True
|
||
time.sleep(HUMAN_POLL_INTERVAL)
|
||
raise LoginTimeoutError(
|
||
f"自动登录失败，{wait_sec}s 内用户在浏览器中也未完成登录"
|
||
)
|
||
|
||
|
||
def _fill_root_login(page) -> bool:
|
||
"""根域登录页（如还在）：fill demo/123456 → click Sign in。返回是否执行了登录。"""
|
||
user_input = page.locator('#jc-user').first
|
||
if user_input.count() == 0:
|
||
return False
|
||
user_input.fill("demo")
|
||
page.locator('#jc-pass').first.fill("123456")
|
||
page.locator('button[type="submit"]:has-text("Sign in")').first.click()
|
||
logger.info("platform_collector: 根域自动登录 demo/123456...")
|
||
try:
|
||
page.wait_for_url(
|
||
lambda u: "/home" in u or "/select" in u or "/industries/" in u,
|
||
timeout=URL_AFTER_LOGIN_TIMEOUT_MS,
|
||
)
|
||
except Exception as e:
|
||
logger.warning("等根域登录跳转超时: %s", e)
|
||
logger.info("platform_collector: 根域登录后 URL=%s", page.url)
|
||
return True
|
||
|
||
|
||
def _click_campus_link(page) -> bool:
|
||
"""在 /select 页面点智慧校园 tile，返回是否点中。"""
|
||
link = page.locator("a[href*='/industries/campus']").first
|
||
if link.count() == 0:
|
||
logger.warning("platform_collector: 未找到 智慧校园 link")
|
||
return False
|
||
link.click()
|
||
logger.info("platform_collector: click 智慧校园 链接")
|
||
try:
|
||
page.wait_for_url(lambda u: "/industries/campus" in u, timeout=SIDEBAR_LINK_TIMEOUT_MS // 2)
|
||
except Exception as e:
|
||
logger.warning("等 sandbox 路径超时: %s；当前 URL=%s", e, page.url)
|
||
return True
|
||
|
||
|
||
def _fill_spa_login(page, settle_ms: int = SPA_LOGIN_SETTLE_MS) -> bool:
|
||
"""sandbox SPA 自动登录（如还在登录页）：fill demo/123456 → click 登录 → 等跳转。
|
||
|
||
返回是否执行了登录（True = 已点登录按钮且 URL 已跳离 login；调用方可继续后续流程）。
|
||
失败时**不抛异常**，由调用方决定是否 fallback 到 _wait_for_manual_login。
|
||
|
||
修复要点（v1.1.5/v1.1.6 登录时间太短导致采集不到数据）：
|
||
- 默认 settle_ms = SPA_LOGIN_SETTLE_MS（4000ms），给 SPA 异步渲染 + token 写入 cookie 时间
|
||
- 等 username input 的 timeout = SPA_LOGIN_INPUT_TIMEOUT_MS（20s）
|
||
- click 按钮后 wait_for_url timeout = SPA_LOGIN_URL_TIMEOUT_MS（30s）
|
||
- 多处过渡路径（/login, /signin, /auth/）一并判断
|
||
"""
|
||
url_lower = page.url.lower()
|
||
if "/login" not in url_lower and "/signin" not in url_lower and "/auth/" not in url_lower:
|
||
return False
|
||
try:
|
||
page.wait_for_selector(
|
||
'input[autocomplete="username"]',
|
||
timeout=SPA_LOGIN_INPUT_TIMEOUT_MS,
|
||
)
|
||
except Exception as e:
|
||
logger.warning("SPA login input 未出现: %s；当前 URL=%s", e, page.url)
|
||
return False
|
||
spa_user = page.locator('input[autocomplete="username"]').first
|
||
if spa_user.count() == 0:
|
||
logger.warning("SPA login input count=0")
|
||
return False
|
||
spa_user.fill("demo")
|
||
page.locator('input[autocomplete="current-password"]').first.fill("123456")
|
||
page.locator(
|
||
'button:has-text("登录"), button[type="submit"]:has-text("Sign in")'
|
||
).first.click()
|
||
logger.info("platform_collector: sandbox SPA 自动登录...")
|
||
page.wait_for_timeout(settle_ms)
|
||
try:
|
||
page.wait_for_url(
|
||
lambda u: "/login" not in u.lower()
|
||
and "/signin" not in u.lower()
|
||
and "/auth/" not in u.lower(),
|
||
timeout=SPA_LOGIN_URL_TIMEOUT_MS,
|
||
)
|
||
except Exception as e:
|
||
logger.warning("SPA 登录超时: %s；当前 URL=%s", e, page.url)
|
||
return False
|
||
logger.info("platform_collector: SPA 登录后 URL=%s", page.url)
|
||
return True
|
||
|
||
|
||
def collect_all(
|
||
limit: int = 0,
|
||
headless: bool = True,
|
||
human_wait_timeout: int = HUMAN_WAIT_TIMEOUT,
|
||
) -> dict:
|
||
"""登录 sandbox 平台 → 抓合同表 + 材料 → 落库。
|
||
|
||
human_wait_timeout: 手工登录 fallback 等待秒数（默认 HUMAN_WAIT_TIMEOUT=180）。
|
||
失败时可能抛出 RPAError（LoginTimeoutError / MissingBrowserError / TableNotFoundError）。
|
||
"""
|
||
try:
|
||
from playwright.sync_api import sync_playwright
|
||
except ImportError:
|
||
logger.error("playwright 未安装，跳过 RPA 采集")
|
||
return _error_result("playwright_not_installed", 0)
|
||
|
||
profile_dir = _safe_getenv_dir("SMARTCAMPUS_PROFILE_DIR", "playwright-profile")
|
||
download_dir = _safe_getenv_dir("MATERIALS_DOWNLOAD_DIR", "materials")
|
||
profile_dir.mkdir(parents=True, exist_ok=True)
|
||
download_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
logger.info(
|
||
"platform_collector: 启动 profile=%s download_dir=%s headless=%s limit=%d "
|
||
"human_wait_timeout=%d",
|
||
profile_dir, download_dir, headless, limit, human_wait_timeout,
|
||
)
|
||
|
||
contracts: list[dict] = []
|
||
fetched_rows = 0
|
||
downloaded = 0
|
||
|
||
with sync_playwright() as p:
|
||
ctx = p.chromium.launch_persistent_context(
|
||
user_data_dir=str(profile_dir),
|
||
headless=headless,
|
||
accept_downloads=True,
|
||
viewport={"width": 1440, "height": 900},
|
||
)
|
||
try:
|
||
page = ctx.new_page()
|
||
# v1.1.6：覆盖 Playwright 默认操作超时（之前无 set_default_timeout，导致
|
||
# 所有 wait_for_selector 默认 30s，调试时不直观）
|
||
page.set_default_timeout(DEFAULT_PAGE_TIMEOUT_MS)
|
||
|
||
# 1. 根域 goto + 根域登录（可选；root login 经常 demo/123456 失败）
|
||
_goto(page, ROOT_URL)
|
||
_fill_root_login(page)
|
||
|
||
# 2. 直接 goto sandbox SPA login（绕过 /select 和 root 跳转）
|
||
logger.info("platform_collector: 强制 goto sandbox SPA login, 当前 URL=%s", page.url)
|
||
_goto(page, SPA_LOGIN_URL)
|
||
# SPA hydration + framework 初始化时间（修复登录时间太短的核心点）
|
||
page.wait_for_timeout(SPA_HYDRATION_MS)
|
||
|
||
# 3. sandbox SPA 自动登录（demo/123456 已知可用）
|
||
# 失败 fallback 手工登录等待（轮询 _is_logged_in + 180s 兜底）
|
||
if not _fill_spa_login(page):
|
||
logger.warning(
|
||
"platform_collector: 自动 fill 失败（URL=%s），fallback 手工登录等待",
|
||
page.url,
|
||
)
|
||
if not headless:
|
||
_wait_for_manual_login(page, wait_sec=human_wait_timeout)
|
||
else:
|
||
# headless 模式无法手工登录：抛 LoginTimeoutError 让 CLI 决定
|
||
raise LoginTimeoutError(
|
||
f"headless 模式下自动 fill 失败，且无法手工登录"
|
||
f"（请用 --show-browser 重试）"
|
||
)
|
||
else:
|
||
# 等 SPA 路由稳定（之前这里没有 wait，直接 goto home 会打断 pending 跳转）
|
||
page.wait_for_timeout(POST_LOGIN_SETTLE_MS)
|
||
|
||
# 4. 强制 goto SPA home（如果还停在 login，重试登录一次）
|
||
for retry in range(SPA_HOME_RETRY_MAX):
|
||
if _is_logged_in(page):
|
||
break
|
||
logger.info(
|
||
"platform_collector: 强制 goto SPA_HOME (retry=%d), 当前 URL=%s",
|
||
retry, page.url,
|
||
)
|
||
_goto(page, SPA_HOME_URL)
|
||
page.wait_for_timeout(SPA_HOME_RETRY_DELAY_MS)
|
||
if _fill_spa_login(page):
|
||
page.wait_for_timeout(POST_LOGIN_SETTLE_MS)
|
||
|
||
# 4.5. retry 后若仍未登录，headless 模式抛超时
|
||
if not _is_logged_in(page):
|
||
if headless:
|
||
raise LoginTimeoutError(
|
||
"SPA_HOME retry 全部失败，URL 仍未离开 login 页"
|
||
)
|
||
_wait_for_manual_login(page, wait_sec=human_wait_timeout)
|
||
|
||
# 5. 等 SPA 渲染 + click 侧边栏 hr/contracts（SPA 路由，避免 goto 触发整页 reload 丢 session）
|
||
try:
|
||
# dump SPA home 一次（用于调试 selector）
|
||
page.wait_for_timeout(POST_LOGIN_SETTLE_MS)
|
||
_dump_page(page, download_dir.parent / "spa_home_dump.html", "SPA home")
|
||
|
||
# 等 SPA 真正加载完（侧边栏 link 出现，最长 SIDEBAR_LINK_TIMEOUT_MS）
|
||
# 修复：之前 15s 太短，sandbox mock 加载慢时直接 fallback 到 pushState → 表格加载不出来
|
||
try:
|
||
page.wait_for_selector(
|
||
"a[href$='/hr/contracts']",
|
||
timeout=SIDEBAR_LINK_TIMEOUT_MS,
|
||
state="attached",
|
||
)
|
||
except Exception as e:
|
||
logger.warning("等侧边栏 hr/contracts link 超时: %s", e)
|
||
|
||
link = page.locator("a[href$='/hr/contracts']").first
|
||
if link.count() > 0:
|
||
link.click()
|
||
logger.info("platform_collector: click 侧边栏 hr/contracts")
|
||
try:
|
||
page.wait_for_url(
|
||
lambda u: "/hr/contracts" in u,
|
||
timeout=SIDEBAR_CLICK_URL_TIMEOUT_MS,
|
||
)
|
||
except Exception as e:
|
||
logger.warning("等 /hr/contracts URL 超时: %s；当前 URL=%s",
|
||
e, page.url)
|
||
else:
|
||
logger.info(
|
||
"platform_collector: 侧边栏 link 仍未出现，"
|
||
"使用 SPA 路由（pushState）到 hr/contracts, 当前 URL=%s",
|
||
page.url,
|
||
)
|
||
# 用 SPA 内部路由而非整页 goto，避免 session 被 server 端踢回 login
|
||
try:
|
||
page.evaluate(
|
||
"u => history.pushState({}, '', u) || "
|
||
"window.dispatchEvent(new PopStateEvent('popstate')),"
|
||
" '/industries/campus/smartcampus/app/hr/contracts'"
|
||
)
|
||
page.wait_for_timeout(PUSHSTATE_SETTLE_MS)
|
||
except Exception as e:
|
||
logger.warning("SPA pushState 失败: %s", e)
|
||
_goto(page, PLATFORM_URL)
|
||
except Exception as e:
|
||
logger.warning("click 侧边栏失败: %s；fallback goto", e)
|
||
_goto(page, PLATFORM_URL)
|
||
|
||
logger.info("platform_collector: 当前 URL=%s", page.url)
|
||
|
||
# 5.5. 进入页面后刷新（SPA 进入新路由时 mock 数据可能未加载，需刷新触发）
|
||
try:
|
||
logger.info("platform_collector: 进入 hr/contracts 后刷新页面")
|
||
page.reload(
|
||
wait_until="networkidle",
|
||
timeout=RELOAD_NETWORKIDLE_TIMEOUT_MS,
|
||
)
|
||
except Exception as e:
|
||
logger.warning("刷新 networkidle 失败: %s；尝试 domcontentloaded", e)
|
||
try:
|
||
page.reload(
|
||
wait_until="domcontentloaded",
|
||
timeout=RELOAD_DOMCONTENTLOADED_TIMEOUT_MS,
|
||
)
|
||
except Exception as e2:
|
||
logger.warning("刷新 domcontentloaded 也失败: %s", e2)
|
||
|
||
# 5.6. 刷新后可能被踢回 login（session 丢失），重新登录
|
||
# 修复：之前没检查 URL，刷新后停在 login 页继续往后走，找不到表格就 dump 报错
|
||
if "/login" in page.url.lower() or "/signin" in page.url.lower():
|
||
logger.warning(
|
||
"platform_collector: 刷新后回到登录页（%s），重新登录",
|
||
page.url,
|
||
)
|
||
_goto(page, SPA_LOGIN_URL)
|
||
page.wait_for_timeout(POST_RELOAD_LOGIN_HYDRATION_MS)
|
||
if not _fill_spa_login(page):
|
||
if not headless:
|
||
_wait_for_manual_login(page, wait_sec=human_wait_timeout)
|
||
else:
|
||
raise LoginTimeoutError(
|
||
"刷新后回到 login 页且 headless 无法手工登录"
|
||
)
|
||
else:
|
||
page.wait_for_timeout(POST_LOGIN_SETTLE_MS)
|
||
_goto(page, PLATFORM_URL)
|
||
page.wait_for_timeout(POST_RELOGIN_GOTO_WAIT_MS)
|
||
|
||
# 7. dump 当前 hr/contracts 页面（用于调试）
|
||
_dump_page(page, download_dir.parent / "hr_contracts_dump.html", "hr/contracts")
|
||
|
||
# 8. 等待表格加载（SPA 异步填充 mock 数据，多等几秒）
|
||
page.wait_for_timeout(TABLE_INITIAL_WAIT_MS) # 给 React 初始化 + mock 数据填充时间
|
||
found_table = False
|
||
for attempt in range(TABLE_POLL_MAX_ATTEMPTS):
|
||
page.wait_for_timeout(TABLE_POLL_INTERVAL_MS)
|
||
cur_url = page.url
|
||
try:
|
||
page.wait_for_selector(
|
||
"table tbody tr",
|
||
timeout=TABLE_TR_TIMEOUT_MS,
|
||
)
|
||
found_table = True
|
||
break
|
||
except Exception:
|
||
logger.info(
|
||
"platform_collector: URL=%s 等待表格加载 (attempt %d/%d)...",
|
||
cur_url, attempt + 1, TABLE_POLL_MAX_ATTEMPTS,
|
||
)
|
||
|
||
if not found_table:
|
||
_dump_page(page, download_dir.parent / "sandbox_dump.html", "sandbox")
|
||
logger.warning("platform_collector: 表格未找到")
|
||
raise TableNotFoundError(
|
||
f"{TABLE_POLL_MAX_ATTEMPTS * TABLE_POLL_INTERVAL_MS // 1000}s "
|
||
f"内未找到表格 (URL={page.url})"
|
||
)
|
||
|
||
# 表格已找到，等 tbody 完全填充 + dump 一份完整 DOM（用于诊断按钮 selector）
|
||
try:
|
||
page.wait_for_function(
|
||
"document.querySelectorAll('table tbody tr').length > 0",
|
||
timeout=DEFAULT_PAGE_TIMEOUT_MS,
|
||
)
|
||
page.wait_for_timeout(FULL_RENDER_WAIT_MS)
|
||
_dump_page(
|
||
page,
|
||
download_dir.parent / "hr_contracts_full_dump.html",
|
||
"完整 hr/contracts",
|
||
)
|
||
except Exception as e:
|
||
logger.warning("等完整渲染超时: %s", e)
|
||
|
||
rows = page.eval_on_selector_all(
|
||
"table tbody tr",
|
||
"rs => rs.map(r => Array.from(r.querySelectorAll('td')).map(td => td.innerText))",
|
||
)
|
||
fetched_rows = len(rows)
|
||
logger.info("platform_collector: 抓到 %d 行", fetched_rows)
|
||
|
||
# 检查每行是否有「续聘材料包」按钮
|
||
try:
|
||
mat_btn_count = page.locator(
|
||
"table tbody tr button:has-text('续聘材料包')"
|
||
).count()
|
||
logger.info(
|
||
"platform_collector: 「续聘材料包」按钮数=%d", mat_btn_count,
|
||
)
|
||
except Exception:
|
||
mat_btn_count = 0
|
||
|
||
# 抓「材料包」card 区段的 DOM（用于后续解析每行 click 后的卡片文本）
|
||
# sandbox 行为：click 按钮后会在表格下方追加「续聘准备任务书」+「单位清单」卡片
|
||
|
||
for i, r in enumerate(rows):
|
||
if limit and i >= limit:
|
||
break
|
||
if len(r) < 7:
|
||
logger.warning("行 %d 列数 < 7（新列序工号+姓名+单位+岗位+合同类型+起止+剩余+预警+建议），跳过: %s", i, r)
|
||
continue
|
||
|
||
# 新列序（sandbox 加了工号 / 岗位 / 合同类型 / 起止日期）：
|
||
# 0: 工号 1: 姓名 2: 单位 3: 岗位 4: 合同类型
|
||
# 5: 起止日期 (含 ~ ) 6: 剩余天数 7: 预警 8: 建议（含原因）
|
||
emp_id = r[0].strip()
|
||
teacher = r[1].strip()
|
||
unit = r[2].strip()
|
||
position = r[3].strip()
|
||
contract_type = r[4].strip()
|
||
period = r[5].strip()
|
||
# 起止日期形如 "2024-03-01 ~ 2026-04-15" → 取 ~ 后
|
||
contract_end = period.split("~")[-1].strip() if "~" in period else period
|
||
days_raw = r[6].strip()
|
||
alert_label = r[7].strip()
|
||
suggestion_hint = r[8].strip() if len(r) > 8 else ""
|
||
|
||
# 用工号生成 contract_id（稳定，不会随姓名变化）
|
||
contract_id = _gen_contract_id(emp_id, contract_end)
|
||
contract = {
|
||
"contract_id": contract_id,
|
||
"teacher_name": teacher,
|
||
"unit_name": unit,
|
||
"contract_end": _normalize_date(contract_end),
|
||
"days_to_expire_hint": _parse_days(days_raw),
|
||
"alert_level_hint": _parse_alert(alert_label),
|
||
"ai_suggestion_hint": suggestion_hint,
|
||
"materials": [],
|
||
}
|
||
|
||
# 点「续聘材料包」按钮 → 等卡片展开 → 抓卡片文本（含 checklist）
|
||
if mat_btn_count > 0:
|
||
try:
|
||
btn = page.locator("table tbody tr").nth(i).locator(
|
||
"button:has-text('续聘材料包')"
|
||
)
|
||
btn.click()
|
||
# 等卡片出现（页面下方追加 h2「续聘准备任务书」/「单位清单」）
|
||
page.wait_for_timeout(CARD_APPEAR_WAIT_MS)
|
||
# 抓表格下方所有 card 文本
|
||
cards_text = page.locator(
|
||
"main .grid > div, main > div > div > div"
|
||
).all_inner_texts()
|
||
materials_text = "\n\n".join(
|
||
t.strip() for t in cards_text if t.strip()
|
||
)
|
||
if materials_text:
|
||
# 存到 materials_content（sandbox mock 卡片文本作为材料包内容）
|
||
contract["materials"].append({
|
||
"kind": "sandbox_card",
|
||
"file_name": f"{contract_id}_materials.txt",
|
||
"text": materials_text,
|
||
})
|
||
downloaded += 1
|
||
logger.info(
|
||
"platform_collector: 抓到卡片文本 %d 字 (%s)",
|
||
len(materials_text), contract_id,
|
||
)
|
||
# dump 一次（i==0）方便调试
|
||
if i == 0:
|
||
_dump_page(
|
||
page,
|
||
download_dir.parent / "after_click_dump.html",
|
||
"首行 click 后",
|
||
)
|
||
except Exception as e:
|
||
logger.warning("抓卡片失败 %s: %s", contract_id, e)
|
||
else:
|
||
if i == 0:
|
||
logger.info(
|
||
"platform_collector: sandbox 未部署「续聘材料包」按钮，"
|
||
"跳过材料抓取（仅采集表格列）",
|
||
)
|
||
|
||
contracts.append(contract)
|
||
finally:
|
||
ctx.close()
|
||
|
||
saved = _persist_contracts(contracts)
|
||
return {
|
||
"platform_url": PLATFORM_URL,
|
||
"fetched_rows": fetched_rows,
|
||
"saved_contracts": saved,
|
||
"downloaded_materials": downloaded,
|
||
"contracts": contracts,
|
||
}
|
||
|
||
|
||
def _persist_contracts(contracts: list[dict]) -> int:
|
||
"""把抓到的合同 + 材料落到 SQLite。仅插入 contracts 表的占位行，
|
||
materials_content 由 parse-materials CLI 后续解析写入。
|
||
|
||
真正的合同台账来自 HR 系统 API（生产）或样例 CSV（本地），
|
||
本采集仅作为"快照"记录抓取时间与材料路径。
|
||
"""
|
||
if not contracts:
|
||
return 0
|
||
|
||
from db.connection import get_connection
|
||
|
||
conn = get_connection()
|
||
try:
|
||
with conn:
|
||
for c in contracts:
|
||
# upsert contracts（仅更新与 sandbox 相关的字段）
|
||
conn.execute(
|
||
"""
|
||
INSERT INTO contracts
|
||
(contract_id, employee_id, teacher_name, unit_id, unit_name,
|
||
position, contract_type, contract_start, contract_end,
|
||
trial_period_end, status, last_material_id, renewed_at)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '可续聘', NULL, NULL)
|
||
ON CONFLICT(contract_id) DO UPDATE SET
|
||
teacher_name = excluded.teacher_name,
|
||
unit_name = excluded.unit_name,
|
||
contract_end = excluded.contract_end
|
||
""",
|
||
(
|
||
c["contract_id"],
|
||
c["contract_id"], # employee_id 暂用 contract_id
|
||
c["teacher_name"],
|
||
"U-RPA", # unit_id 占位
|
||
c["unit_name"],
|
||
"（RPA 采集）",
|
||
"（RPA 采集）",
|
||
"1970-01-01",
|
||
c["contract_end"],
|
||
c["contract_end"],
|
||
),
|
||
)
|
||
|
||
# 写 materials_content（sandbox card 文本 + parse-materials 解析后文件路径）
|
||
from db.migrations import ensure_materials_content_schema
|
||
|
||
ensure_materials_content_schema(conn)
|
||
|
||
for m in c["materials"]:
|
||
if isinstance(m, dict) and m.get("kind") == "sandbox_card":
|
||
# sandbox mock 卡片文本，直接写入 extracted_text
|
||
conn.execute(
|
||
"""
|
||
INSERT OR REPLACE INTO materials_content
|
||
(contract_id, file_name, file_path, extracted_text,
|
||
parse_method, parsed_at)
|
||
VALUES (?, ?, ?, ?, 'sandbox_card', ?)
|
||
""",
|
||
(
|
||
c["contract_id"],
|
||
m["file_name"],
|
||
f"sandbox://hr/contracts/{c['contract_id']}",
|
||
m["text"],
|
||
datetime.now().isoformat(timespec="seconds"),
|
||
),
|
||
)
|
||
# 同步解析 → 抽 year + 等次 → 写入 performance_records
|
||
perf_n = repo.parse_sandbox_card_performance(
|
||
conn, c["contract_id"], m["text"],
|
||
)
|
||
if perf_n > 0:
|
||
logger.info(
|
||
"platform_collector: %s 解析卡片 → performance %d 条",
|
||
c["contract_id"], perf_n,
|
||
)
|
||
else:
|
||
# 本地文件路径（parse-materials 解析后写入）
|
||
fp = m if isinstance(m, str) else m.get("file_path")
|
||
fname = Path(fp).name if fp else "unknown"
|
||
conn.execute(
|
||
"""
|
||
INSERT OR REPLACE INTO materials_content
|
||
(contract_id, file_name, file_path, extracted_text,
|
||
parse_method, parsed_at)
|
||
VALUES (?, ?, ?, NULL, 'pending', ?)
|
||
""",
|
||
(
|
||
c["contract_id"],
|
||
fname,
|
||
fp,
|
||
datetime.now().isoformat(timespec="seconds"),
|
||
),
|
||
)
|
||
return len(contracts)
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def _error_result(reason: str, fetched: int) -> dict:
|
||
return {
|
||
"platform_url": PLATFORM_URL,
|
||
"fetched_rows": fetched,
|
||
"saved_contracts": 0,
|
||
"downloaded_materials": 0,
|
||
"contracts": [],
|
||
"error": reason,
|
||
}
|
||
|
||
|
||
def _gen_contract_id(emp_id: str, end_date: str) -> str:
|
||
"""从 (工号, 到期日) 派 contract_id，确保幂等。
|
||
|
||
sandbox v1.1.1 起改用「工号」作主键（之前用姓名）。
|
||
"""
|
||
safe_emp = re.sub(r"[^\w]", "", emp_id)[:10] or "T"
|
||
safe_end = end_date.replace("-", "")[:8]
|
||
return f"RPA-{safe_emp}-{safe_end}"
|
||
|
||
|
||
def _normalize_date(s: str) -> str:
|
||
"""尽量把 sandbox 返回的日期归一为 YYYY-MM-DD。"""
|
||
s = s.strip()
|
||
if re.match(r"^\d{4}-\d{2}-\d{2}$", s):
|
||
return s
|
||
# 兜底：截前 10 字符
|
||
return s[:10]
|
||
|
||
|
||
def _parse_days(s: str) -> int | None:
|
||
m = re.search(r"-?\d+", s)
|
||
return int(m.group(0)) if m else None
|
||
|
||
|
||
def _parse_alert(s: str) -> str:
|
||
s = s.strip()
|
||
if "红" in s:
|
||
return "red"
|
||
if "橙" in s:
|
||
return "orange"
|
||
if "黄" in s:
|
||
return "yellow"
|
||
return "unknown" |