Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 6437d6e414 |
@@ -51,6 +51,14 @@ MATERIALS_DOWNLOAD_DIR=
|
||||
# OCR 语言(pytesseract:默认 chi_sim+eng)
|
||||
OCR_LANG=chi_sim+eng
|
||||
|
||||
# ----- v1.1.6:RPA 超时覆盖(与 skill-template human_verification 对齐)-----
|
||||
# 手工登录等待超时(秒,默认 180 = HUMAN_WAIT_TIMEOUT)
|
||||
HUMAN_WAIT_TIMEOUT=
|
||||
# Playwright 默认操作超时(毫秒,默认 15000)
|
||||
RPA_PAGE_TIMEOUT_MS=
|
||||
# page.goto 导航超时(毫秒,默认 60000)
|
||||
RPA_GOTO_TIMEOUT_MS=
|
||||
|
||||
# ----- v1.1.1:AI 决策(宿主 LLM) -----
|
||||
# 宿主 jiangchang_skill_core.llm_manager 调 LLM 时,模型名覆盖(可选)
|
||||
LLM_MODEL_NAME=
|
||||
|
||||
12
CHANGELOG.md
12
CHANGELOG.md
@@ -2,6 +2,18 @@
|
||||
|
||||
本文件对应技能市场 **「更新日志」** Tab。
|
||||
|
||||
## 1.1.6
|
||||
|
||||
- **RPA 重构(按 skill-template `examples/real_browser_rpa` 设计模式)**
|
||||
- **超时参数统一抽到 `constants.py`**:新增 `HUMAN_WAIT_TIMEOUT=180` / `HUMAN_POLL_INTERVAL=2.0` / `DEFAULT_PAGE_TIMEOUT_MS=15s` / `GOTO_TIMEOUT_MS=60s` / `SPA_LOGIN_*` / `TABLE_*` / `SIDEBAR_LINK_TIMEOUT_MS` 等 24 个常量(与 skill-template human_verification 同名常量对齐)
|
||||
- **新增手工登录 fallback**:自动 fill demo/123456 失败时自动切到 `_wait_for_manual_login`(轮询 `_is_logged_in` 每 2s 一次,最多 `HUMAN_WAIT_TIMEOUT` 秒,超时抛 `LoginTimeoutError`),给用户手工登录机会(与 skill-template `task_rpa.py::_ensure_logged_in` 设计对齐)
|
||||
- **结构化错误码**:`RPAError` 基类 + `LoginTimeoutError`(`ERROR:LOGIN_TIMEOUT`)/ `MissingBrowserError`(`ERROR:MISSING_BROWSER`)/ `TableNotFoundError`(`ERROR:TABLE_NOT_FOUND`),供 CLI/宿主解析
|
||||
- **CLI 新增 `--human-wait-timeout N` flag**
|
||||
- **`_goto` 默认 timeout 改用 `GOTO_TIMEOUT_MS=60s`**;新增 `page.set_default_timeout(DEFAULT_PAGE_TIMEOUT_MS)`
|
||||
- **依赖清单修复**:删除 `requirements.txt` 中违规的 `playwright>=1.40.0`(playwright 由宿主 `jiangchang_skill_core` runtime 提供,文件顶部注释已声明)
|
||||
- **环境变量**:`.env.example` 新增 `HUMAN_WAIT_TIMEOUT` / `RPA_PAGE_TIMEOUT_MS` / `RPA_GOTO_TIMEOUT_MS` 覆盖项
|
||||
- **保留 v1.1.5 自动 fill 优势**:sandbox demo/123456 已知凭据自动登录(生产环境切换无 demo 凭据时自动 fallback 手工)
|
||||
|
||||
## 1.1.5
|
||||
|
||||
- **platform_collector RPA 采集修复(登录等待时间太短 → 拿不到数据)**
|
||||
|
||||
2
SKILL.md
2
SKILL.md
@@ -1,7 +1,7 @@
|
||||
---
|
||||
name: 合同到期智能预警与续聘辅助
|
||||
description: "每日凌晨扫描全校教职工合同电子台账,对距离到期日 90/60/30 天的合同分级预警,并自动生成《续聘准备任务书》和《本月合同到期人员清单》。"
|
||||
version: 1.1.5
|
||||
version: 1.1.6
|
||||
author: 深圳匠厂科技有限公司
|
||||
metadata:
|
||||
openclaw:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# 合同到期智能预警与续聘辅助 - 技能特有依赖
|
||||
# jiangchang-platform-kit 等公共能力由宿主共享 runtime 提供,不要写入此文件
|
||||
|
||||
# 公共依赖由宿主共享 runtime 提供(jiangchang-platform-kit、playwright 等);
|
||||
# 这里只声明技能特有 Python 依赖。
|
||||
# HTTP 请求
|
||||
requests>=2.31.0,<3
|
||||
|
||||
@@ -10,9 +10,6 @@ python-dateutil>=2.8.0,<3
|
||||
# HTML 模板渲染(个人版任务书)
|
||||
jinja2>=3.1.0,<4
|
||||
|
||||
# v1.1.1:SmartCampus HR 平台 RPA 采集
|
||||
playwright>=1.40.0,<2
|
||||
|
||||
# v1.1.1:续聘材料包 PDF/Word 文本提取
|
||||
pypdf>=4.0.0,<5
|
||||
python-docx>=1.1.0,<2
|
||||
|
||||
@@ -21,7 +21,7 @@ from typing import Callable, List, Optional
|
||||
from db.connection import get_connection, init_db
|
||||
from db import repository as repo
|
||||
from service import dispatcher, html_renderer, material_parser, platform_collector, scanner, task_generator
|
||||
from util.constants import DEFAULT_THRESHOLDS, SKILL_VERSION
|
||||
from util.constants import DEFAULT_THRESHOLDS, HUMAN_WAIT_TIMEOUT, SKILL_VERSION
|
||||
from util.logging_setup import get_skill_logger, setup_skill_logging
|
||||
from util.runtime_paths import (
|
||||
get_db_path,
|
||||
@@ -204,12 +204,25 @@ def cmd_collect_platform(args: argparse.Namespace) -> int:
|
||||
退出码:
|
||||
0 = 成功(即使抓到 0 行也返回 0,错误信息在 output["error"])
|
||||
1 = 通用异常
|
||||
2 = RPA 结构化错误(LoginTimeout / MissingBrowser / TableNotFound 等)
|
||||
"""
|
||||
try:
|
||||
result = platform_collector.collect_all(
|
||||
limit=args.limit,
|
||||
headless=not args.show_browser,
|
||||
human_wait_timeout=args.human_wait_timeout or HUMAN_WAIT_TIMEOUT,
|
||||
)
|
||||
except platform_collector.RPAError as e:
|
||||
# v1.1.6:结构化错误(ERROR:LOGIN_TIMEOUT 等)→ exit 2
|
||||
print(json.dumps({
|
||||
"error": e.code,
|
||||
"detail": str(e),
|
||||
"fetched_rows": 0,
|
||||
"saved_contracts": 0,
|
||||
"downloaded_materials": 0,
|
||||
"contracts": [],
|
||||
}, ensure_ascii=False, indent=2))
|
||||
return 2
|
||||
except Exception as e:
|
||||
print(json.dumps({"error": "exception", "detail": str(e)},
|
||||
ensure_ascii=False, indent=2))
|
||||
@@ -333,6 +346,8 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
help="限制抓取行数(0=全量)")
|
||||
sp.add_argument("--show-browser", action="store_true",
|
||||
help="显示浏览器(首次手工登录用,默认 headless)")
|
||||
sp.add_argument("--human-wait-timeout", type=int, default=None,
|
||||
help="手工登录 fallback 等待秒数(默认 180 = HUMAN_WAIT_TIMEOUT)")
|
||||
sp.set_defaults(handler=cmd_collect_platform)
|
||||
|
||||
sp = sub.add_parser(
|
||||
|
||||
@@ -1,23 +1,23 @@
|
||||
"""Playwright RPA:从 SmartCampus HR 平台抓合同表 + 续聘材料包。
|
||||
|
||||
设计要点:
|
||||
设计要点(v1.1.6,按 skill-template examples/real_browser_rpa 设计模式):
|
||||
- 使用 launch_persistent_context 复用浏览器登录态,避免密码硬编码
|
||||
- 首次使用需用户手工登录一次(CLI 默认 --headless false 便于交互)
|
||||
- 后续使用无头模式(profile 已存 cookie)
|
||||
- **登录策略**:自动 fill demo/123456(sandbox 已知凭据) + 失败时 fallback 手工登录等待(轮询 180s)
|
||||
- 抓表格:DOM eval_on_selector_all → JSON
|
||||
- 抓材料:点 "续聘材料包" 按钮 → expect_download → 保存到本地
|
||||
- 抓材料:点 "续聘材料包" 按钮 → 抓卡片 inner_text
|
||||
|
||||
依赖为可选:playwright 未安装时 import 抛错,上层 CLI 捕获并降级报错。
|
||||
|
||||
导航流程(v1.1.1 强化版 — 强制 goto):
|
||||
1. goto 根域
|
||||
2. 根域 login(如仍显示表单)→ 等跳到 /home 或 /select
|
||||
3. 强制 goto /select
|
||||
4. 在 /select 页面 click 智慧校园 tile → 等 URL 含 /industries/campus
|
||||
5. 若 URL 含 /login,填 sandbox SPA 登录表单 → 等跳走
|
||||
6. 强制 goto SPA_HOME
|
||||
7. click 侧边栏 hr/contracts
|
||||
8. 等 table tbody tr
|
||||
3. 强制 goto sandbox SPA login URL
|
||||
4. SPA login:自动 fill demo/123456,失败 fallback 手工等待(HUMAN_WAIT_TIMEOUT=180s)
|
||||
5. 强制 goto SPA_HOME
|
||||
6. click 侧边栏 hr/contracts
|
||||
7. 等 table tbody tr(最长 ~40s)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -25,11 +25,38 @@ from __future__ import annotations
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from db import repository as repo
|
||||
from util.constants import SKILL_SLUG
|
||||
from util.constants import (
|
||||
CARD_APPEAR_WAIT_MS,
|
||||
DEFAULT_PAGE_TIMEOUT_MS,
|
||||
FULL_RENDER_WAIT_MS,
|
||||
GOTO_TIMEOUT_MS,
|
||||
HUMAN_POLL_INTERVAL,
|
||||
HUMAN_WAIT_TIMEOUT,
|
||||
POST_LOGIN_SETTLE_MS,
|
||||
POST_RELOAD_LOGIN_HYDRATION_MS,
|
||||
POST_RELOGIN_GOTO_WAIT_MS,
|
||||
PUSHSTATE_SETTLE_MS,
|
||||
RELOAD_DOMCONTENTLOADED_TIMEOUT_MS,
|
||||
RELOAD_NETWORKIDLE_TIMEOUT_MS,
|
||||
SIDEBAR_CLICK_URL_TIMEOUT_MS,
|
||||
SIDEBAR_LINK_TIMEOUT_MS,
|
||||
SPA_HOME_RETRY_DELAY_MS,
|
||||
SPA_HOME_RETRY_MAX,
|
||||
SPA_HYDRATION_MS,
|
||||
SPA_LOGIN_INPUT_TIMEOUT_MS,
|
||||
SPA_LOGIN_SETTLE_MS,
|
||||
SPA_LOGIN_URL_TIMEOUT_MS,
|
||||
TABLE_INITIAL_WAIT_MS,
|
||||
TABLE_POLL_INTERVAL_MS,
|
||||
TABLE_POLL_MAX_ATTEMPTS,
|
||||
TABLE_TR_TIMEOUT_MS,
|
||||
URL_AFTER_LOGIN_TIMEOUT_MS,
|
||||
)
|
||||
from util.logging_setup import get_skill_logger
|
||||
from util.runtime_paths import ensure_dirs
|
||||
|
||||
@@ -47,6 +74,47 @@ SPA_HOME_URL = PLATFORM_BASE + "/industries/campus/smartcampus/app/home"
|
||||
SPA_LOGIN_URL = PLATFORM_BASE + "/industries/campus/smartcampus/login"
|
||||
|
||||
|
||||
# ---------- 错误码(与 skill-template 对齐) ----------
|
||||
|
||||
class RPAError(Exception):
|
||||
"""RPA 流程错误基类。"""
|
||||
|
||||
code: str = "RPA_ERROR"
|
||||
|
||||
def __init__(self, message: str):
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
|
||||
|
||||
class LoginTimeoutError(RPAError):
|
||||
"""手工登录超时(HUMAN_WAIT_TIMEOUT 秒内未检测到登录成功)。
|
||||
|
||||
错误码:ERROR:LOGIN_TIMEOUT(与 skill-template task_rpa.py 对齐)
|
||||
"""
|
||||
|
||||
code = "ERROR:LOGIN_TIMEOUT"
|
||||
|
||||
|
||||
class MissingBrowserError(RPAError):
|
||||
"""未检测到 Chrome/Edge。
|
||||
|
||||
错误码:ERROR:MISSING_BROWSER(与 skill-template browser_session.py 对齐)
|
||||
"""
|
||||
|
||||
code = "ERROR:MISSING_BROWSER"
|
||||
|
||||
|
||||
class TableNotFoundError(RPAError):
|
||||
"""40s 内未找到表格。
|
||||
|
||||
错误码:ERROR:TABLE_NOT_FOUND
|
||||
"""
|
||||
|
||||
code = "ERROR:TABLE_NOT_FOUND"
|
||||
|
||||
|
||||
# ---------- 工具函数 ----------
|
||||
|
||||
def _safe_getenv_dir(env_key: str, default_rel: str) -> Path:
|
||||
raw = os.environ.get(env_key)
|
||||
if raw:
|
||||
@@ -56,7 +124,7 @@ def _safe_getenv_dir(env_key: str, default_rel: str) -> Path:
|
||||
return (get_data_dir() / default_rel).resolve()
|
||||
|
||||
|
||||
def _goto(page, url: str, timeout: int = 20_000) -> None:
|
||||
def _goto(page, url: str, timeout: int = GOTO_TIMEOUT_MS) -> None:
|
||||
"""带 networkidle / domcontentloaded 双兜底的 goto。"""
|
||||
try:
|
||||
page.goto(url, wait_until="networkidle", timeout=timeout)
|
||||
@@ -82,6 +150,47 @@ def _dump_page(page, path: Path, label: str) -> None:
|
||||
logger.warning("dump %s 失败: %s", label, e)
|
||||
|
||||
|
||||
def _is_logged_in(page) -> bool:
|
||||
"""检测 sandbox SPA 是否已登录(URL 含 /app/ 且不含 /login 或 /signin)。
|
||||
|
||||
与 skill-template real_browser_rpa _is_logged_in 概念对齐(这里只看 URL,
|
||||
因为 sandbox SPA 登录后 URL 稳定跳到 /app/...)。
|
||||
"""
|
||||
url_lower = page.url.lower()
|
||||
return (
|
||||
"/app/" in url_lower
|
||||
and "/login" not in url_lower
|
||||
and "/signin" not in url_lower
|
||||
)
|
||||
|
||||
|
||||
def _wait_for_manual_login(page, wait_sec: int = HUMAN_WAIT_TIMEOUT) -> bool:
|
||||
"""手工登录 fallback:轮询 _is_logged_in,每 HUMAN_POLL_INTERVAL 秒一次。
|
||||
|
||||
与 skill-template task_rpa.py::_ensure_logged_in 设计对齐:
|
||||
- 不自动破解,等用户在浏览器手工完成登录
|
||||
- 屏幕打印 [登录] 提示
|
||||
- 超时抛 LoginTimeoutError(ERROR:LOGIN_TIMEOUT)
|
||||
|
||||
自动 fill demo/123456 失败时调用此函数。
|
||||
"""
|
||||
print(
|
||||
f"[登录] 自动登录失败,请在浏览器中完成登录"
|
||||
f"(sandbox demo/123456 或真实账号),最多等待 {wait_sec} 秒..."
|
||||
)
|
||||
deadline = time.monotonic() + wait_sec
|
||||
attempt = 0
|
||||
while time.monotonic() < deadline:
|
||||
attempt += 1
|
||||
if _is_logged_in(page):
|
||||
print(f"[登录] 检测到登录成功(attempt {attempt})")
|
||||
return True
|
||||
time.sleep(HUMAN_POLL_INTERVAL)
|
||||
raise LoginTimeoutError(
|
||||
f"自动登录失败,{wait_sec}s 内用户在浏览器中也未完成登录"
|
||||
)
|
||||
|
||||
|
||||
def _fill_root_login(page) -> bool:
|
||||
"""根域登录页(如还在):fill demo/123456 → click Sign in。返回是否执行了登录。"""
|
||||
user_input = page.locator('#jc-user').first
|
||||
@@ -94,7 +203,7 @@ def _fill_root_login(page) -> bool:
|
||||
try:
|
||||
page.wait_for_url(
|
||||
lambda u: "/home" in u or "/select" in u or "/industries/" in u,
|
||||
timeout=20_000,
|
||||
timeout=URL_AFTER_LOGIN_TIMEOUT_MS,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("等根域登录跳转超时: %s", e)
|
||||
@@ -111,28 +220,32 @@ def _click_campus_link(page) -> bool:
|
||||
link.click()
|
||||
logger.info("platform_collector: click 智慧校园 链接")
|
||||
try:
|
||||
page.wait_for_url(lambda u: "/industries/campus" in u, timeout=15_000)
|
||||
page.wait_for_url(lambda u: "/industries/campus" in u, timeout=SIDEBAR_LINK_TIMEOUT_MS // 2)
|
||||
except Exception as e:
|
||||
logger.warning("等 sandbox 路径超时: %s;当前 URL=%s", e, page.url)
|
||||
return True
|
||||
|
||||
|
||||
def _fill_spa_login(page, settle_ms: int = 4000) -> bool:
|
||||
"""sandbox SPA 登录(如还在登录页):fill demo/123456 → click 登录 → 等跳转。
|
||||
def _fill_spa_login(page, settle_ms: int = SPA_LOGIN_SETTLE_MS) -> bool:
|
||||
"""sandbox SPA 自动登录(如还在登录页):fill demo/123456 → click 登录 → 等跳转。
|
||||
|
||||
返回是否执行了登录(True = 已点登录按钮;调用方应继续等 URL 跳走)。
|
||||
返回是否执行了登录(True = 已点登录按钮且 URL 已跳离 login;调用方可继续后续流程)。
|
||||
失败时**不抛异常**,由调用方决定是否 fallback 到 _wait_for_manual_login。
|
||||
|
||||
修复要点(登录时间太短导致采集不到数据):
|
||||
- 默认 settle_ms 从 0 提到 4000ms,给 SPA 异步渲染 + token 写入 cookie 时间
|
||||
- 等 username input 的 timeout 从 10s 提到 20s(mock 慢启动)
|
||||
- click 按钮后显式 wait_for_url 含更长 timeout(30s),URL 真的不含 /login 才返回 True
|
||||
- 多处过渡路径(/login, /signin, /auth/)一并判断,避免 SPA 中间态被误判为「已离开登录页」
|
||||
修复要点(v1.1.5/v1.1.6 登录时间太短导致采集不到数据):
|
||||
- 默认 settle_ms = SPA_LOGIN_SETTLE_MS(4000ms),给 SPA 异步渲染 + token 写入 cookie 时间
|
||||
- 等 username input 的 timeout = SPA_LOGIN_INPUT_TIMEOUT_MS(20s)
|
||||
- click 按钮后 wait_for_url timeout = SPA_LOGIN_URL_TIMEOUT_MS(30s)
|
||||
- 多处过渡路径(/login, /signin, /auth/)一并判断
|
||||
"""
|
||||
url_lower = page.url.lower()
|
||||
if "/login" not in url_lower and "/signin" not in url_lower and "/auth/" not in url_lower:
|
||||
return False
|
||||
try:
|
||||
page.wait_for_selector('input[autocomplete="username"]', timeout=20_000)
|
||||
page.wait_for_selector(
|
||||
'input[autocomplete="username"]',
|
||||
timeout=SPA_LOGIN_INPUT_TIMEOUT_MS,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("SPA login input 未出现: %s;当前 URL=%s", e, page.url)
|
||||
return False
|
||||
@@ -146,15 +259,13 @@ def _fill_spa_login(page, settle_ms: int = 4000) -> bool:
|
||||
'button:has-text("登录"), button[type="submit"]:has-text("Sign in")'
|
||||
).first.click()
|
||||
logger.info("platform_collector: sandbox SPA 自动登录...")
|
||||
# click 后给 SPA 一点时间把 token 写入 cookie / 触发跳转
|
||||
page.wait_for_timeout(settle_ms)
|
||||
# 再等到 URL 真的跳离 login 页(最长 30s;之前 20s 太短)
|
||||
try:
|
||||
page.wait_for_url(
|
||||
lambda u: "/login" not in u.lower()
|
||||
and "/signin" not in u.lower()
|
||||
and "/auth/" not in u.lower(),
|
||||
timeout=30_000,
|
||||
timeout=SPA_LOGIN_URL_TIMEOUT_MS,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("SPA 登录超时: %s;当前 URL=%s", e, page.url)
|
||||
@@ -163,8 +274,16 @@ def _fill_spa_login(page, settle_ms: int = 4000) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def collect_all(limit: int = 0, headless: bool = True) -> dict:
|
||||
"""登录 sandbox 平台 → 抓合同表 + 材料 → 落库。"""
|
||||
def collect_all(
|
||||
limit: int = 0,
|
||||
headless: bool = True,
|
||||
human_wait_timeout: int = HUMAN_WAIT_TIMEOUT,
|
||||
) -> dict:
|
||||
"""登录 sandbox 平台 → 抓合同表 + 材料 → 落库。
|
||||
|
||||
human_wait_timeout: 手工登录 fallback 等待秒数(默认 HUMAN_WAIT_TIMEOUT=180)。
|
||||
失败时可能抛出 RPAError(LoginTimeoutError / MissingBrowserError / TableNotFoundError)。
|
||||
"""
|
||||
try:
|
||||
from playwright.sync_api import sync_playwright
|
||||
except ImportError:
|
||||
@@ -177,8 +296,9 @@ def collect_all(limit: int = 0, headless: bool = True) -> dict:
|
||||
download_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.info(
|
||||
"platform_collector: 启动 profile=%s download_dir=%s headless=%s limit=%d",
|
||||
profile_dir, download_dir, headless, limit,
|
||||
"platform_collector: 启动 profile=%s download_dir=%s headless=%s limit=%d "
|
||||
"human_wait_timeout=%d",
|
||||
profile_dir, download_dir, headless, limit, human_wait_timeout,
|
||||
)
|
||||
|
||||
contracts: list[dict] = []
|
||||
@@ -194,6 +314,9 @@ def collect_all(limit: int = 0, headless: bool = True) -> dict:
|
||||
)
|
||||
try:
|
||||
page = ctx.new_page()
|
||||
# v1.1.6:覆盖 Playwright 默认操作超时(之前无 set_default_timeout,导致
|
||||
# 所有 wait_for_selector 默认 30s,调试时不直观)
|
||||
page.set_default_timeout(DEFAULT_PAGE_TIMEOUT_MS)
|
||||
|
||||
# 1. 根域 goto + 根域登录(可选;root login 经常 demo/123456 失败)
|
||||
_goto(page, ROOT_URL)
|
||||
@@ -203,38 +326,61 @@ def collect_all(limit: int = 0, headless: bool = True) -> dict:
|
||||
logger.info("platform_collector: 强制 goto sandbox SPA login, 当前 URL=%s", page.url)
|
||||
_goto(page, SPA_LOGIN_URL)
|
||||
# SPA hydration + framework 初始化时间(修复登录时间太短的核心点)
|
||||
page.wait_for_timeout(3000)
|
||||
page.wait_for_timeout(SPA_HYDRATION_MS)
|
||||
|
||||
# 3. sandbox SPA 登录(demo/123456 已知可用)
|
||||
# 等待 URL 真的跳离 login(settle_ms 已给到 4s,wait_for_url 又给 30s)
|
||||
if _fill_spa_login(page):
|
||||
# 3. sandbox SPA 自动登录(demo/123456 已知可用)
|
||||
# 失败 fallback 手工登录等待(轮询 _is_logged_in + 180s 兜底)
|
||||
if not _fill_spa_login(page):
|
||||
logger.warning(
|
||||
"platform_collector: 自动 fill 失败(URL=%s),fallback 手工登录等待",
|
||||
page.url,
|
||||
)
|
||||
if not headless:
|
||||
_wait_for_manual_login(page, wait_sec=human_wait_timeout)
|
||||
else:
|
||||
# headless 模式无法手工登录:抛 LoginTimeoutError 让 CLI 决定
|
||||
raise LoginTimeoutError(
|
||||
f"headless 模式下自动 fill 失败,且无法手工登录"
|
||||
f"(请用 --show-browser 重试)"
|
||||
)
|
||||
else:
|
||||
# 等 SPA 路由稳定(之前这里没有 wait,直接 goto home 会打断 pending 跳转)
|
||||
page.wait_for_timeout(2000)
|
||||
page.wait_for_timeout(POST_LOGIN_SETTLE_MS)
|
||||
|
||||
# 4. 强制 goto SPA home(如果还停在 login,重试登录一次)
|
||||
for retry in range(2):
|
||||
if "/app/" in page.url and "/login" not in page.url.lower():
|
||||
for retry in range(SPA_HOME_RETRY_MAX):
|
||||
if _is_logged_in(page):
|
||||
break
|
||||
logger.info(
|
||||
"platform_collector: 强制 goto SPA_HOME (retry=%d), 当前 URL=%s",
|
||||
retry, page.url,
|
||||
)
|
||||
_goto(page, SPA_HOME_URL)
|
||||
page.wait_for_timeout(2500)
|
||||
page.wait_for_timeout(SPA_HOME_RETRY_DELAY_MS)
|
||||
if _fill_spa_login(page):
|
||||
page.wait_for_timeout(2000)
|
||||
page.wait_for_timeout(POST_LOGIN_SETTLE_MS)
|
||||
|
||||
# 4.5. retry 后若仍未登录,headless 模式抛超时
|
||||
if not _is_logged_in(page):
|
||||
if headless:
|
||||
raise LoginTimeoutError(
|
||||
"SPA_HOME retry 全部失败,URL 仍未离开 login 页"
|
||||
)
|
||||
_wait_for_manual_login(page, wait_sec=human_wait_timeout)
|
||||
|
||||
# 5. 等 SPA 渲染 + click 侧边栏 hr/contracts(SPA 路由,避免 goto 触发整页 reload 丢 session)
|
||||
try:
|
||||
# dump SPA home 一次(用于调试 selector)
|
||||
page.wait_for_timeout(2000)
|
||||
page.wait_for_timeout(POST_LOGIN_SETTLE_MS)
|
||||
_dump_page(page, download_dir.parent / "spa_home_dump.html", "SPA home")
|
||||
|
||||
# 等 SPA 真正加载完(侧边栏 link 出现,最长 30s)
|
||||
# 等 SPA 真正加载完(侧边栏 link 出现,最长 SIDEBAR_LINK_TIMEOUT_MS)
|
||||
# 修复:之前 15s 太短,sandbox mock 加载慢时直接 fallback 到 pushState → 表格加载不出来
|
||||
try:
|
||||
page.wait_for_selector(
|
||||
"a[href$='/hr/contracts']", timeout=30_000, state="attached"
|
||||
"a[href$='/hr/contracts']",
|
||||
timeout=SIDEBAR_LINK_TIMEOUT_MS,
|
||||
state="attached",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("等侧边栏 hr/contracts link 超时: %s", e)
|
||||
@@ -245,7 +391,8 @@ def collect_all(limit: int = 0, headless: bool = True) -> dict:
|
||||
logger.info("platform_collector: click 侧边栏 hr/contracts")
|
||||
try:
|
||||
page.wait_for_url(
|
||||
lambda u: "/hr/contracts" in u, timeout=10_000
|
||||
lambda u: "/hr/contracts" in u,
|
||||
timeout=SIDEBAR_CLICK_URL_TIMEOUT_MS,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("等 /hr/contracts URL 超时: %s;当前 URL=%s",
|
||||
@@ -263,7 +410,7 @@ def collect_all(limit: int = 0, headless: bool = True) -> dict:
|
||||
"window.dispatchEvent(new PopStateEvent('popstate')),"
|
||||
" '/industries/campus/smartcampus/app/hr/contracts'"
|
||||
)
|
||||
page.wait_for_timeout(1500)
|
||||
page.wait_for_timeout(PUSHSTATE_SETTLE_MS)
|
||||
except Exception as e:
|
||||
logger.warning("SPA pushState 失败: %s", e)
|
||||
_goto(page, PLATFORM_URL)
|
||||
@@ -276,15 +423,21 @@ def collect_all(limit: int = 0, headless: bool = True) -> dict:
|
||||
# 5.5. 进入页面后刷新(SPA 进入新路由时 mock 数据可能未加载,需刷新触发)
|
||||
try:
|
||||
logger.info("platform_collector: 进入 hr/contracts 后刷新页面")
|
||||
page.reload(wait_until="networkidle", timeout=30_000)
|
||||
page.reload(
|
||||
wait_until="networkidle",
|
||||
timeout=RELOAD_NETWORKIDLE_TIMEOUT_MS,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("刷新 networkidle 失败: %s;尝试 domcontentloaded", e)
|
||||
try:
|
||||
page.reload(wait_until="domcontentloaded", timeout=30_000)
|
||||
page.reload(
|
||||
wait_until="domcontentloaded",
|
||||
timeout=RELOAD_DOMCONTENTLOADED_TIMEOUT_MS,
|
||||
)
|
||||
except Exception as e2:
|
||||
logger.warning("刷新 domcontentloaded 也失败: %s", e2)
|
||||
|
||||
# 5.6. 刷新后可能被踢回 login(session 丢失),重新登录一次
|
||||
# 5.6. 刷新后可能被踢回 login(session 丢失),重新登录
|
||||
# 修复:之前没检查 URL,刷新后停在 login 页继续往后走,找不到表格就 dump 报错
|
||||
if "/login" in page.url.lower() or "/signin" in page.url.lower():
|
||||
logger.warning(
|
||||
@@ -292,43 +445,56 @@ def collect_all(limit: int = 0, headless: bool = True) -> dict:
|
||||
page.url,
|
||||
)
|
||||
_goto(page, SPA_LOGIN_URL)
|
||||
page.wait_for_timeout(3000)
|
||||
if _fill_spa_login(page):
|
||||
page.wait_for_timeout(2000)
|
||||
page.wait_for_timeout(POST_RELOAD_LOGIN_HYDRATION_MS)
|
||||
if not _fill_spa_login(page):
|
||||
if not headless:
|
||||
_wait_for_manual_login(page, wait_sec=human_wait_timeout)
|
||||
else:
|
||||
raise LoginTimeoutError(
|
||||
"刷新后回到 login 页且 headless 无法手工登录"
|
||||
)
|
||||
else:
|
||||
page.wait_for_timeout(POST_LOGIN_SETTLE_MS)
|
||||
_goto(page, PLATFORM_URL)
|
||||
page.wait_for_timeout(2000)
|
||||
page.wait_for_timeout(POST_RELOGIN_GOTO_WAIT_MS)
|
||||
|
||||
# 7. dump 当前 hr/contracts 页面(用于调试)
|
||||
_dump_page(page, download_dir.parent / "hr_contracts_dump.html", "hr/contracts")
|
||||
|
||||
# 8. 等待表格加载(SPA 异步填充 mock 数据,多等几秒)
|
||||
page.wait_for_timeout(3000) # 给 React 初始化 + mock 数据填充时间
|
||||
page.wait_for_timeout(TABLE_INITIAL_WAIT_MS) # 给 React 初始化 + mock 数据填充时间
|
||||
found_table = False
|
||||
for attempt in range(20):
|
||||
page.wait_for_timeout(2000)
|
||||
for attempt in range(TABLE_POLL_MAX_ATTEMPTS):
|
||||
page.wait_for_timeout(TABLE_POLL_INTERVAL_MS)
|
||||
cur_url = page.url
|
||||
try:
|
||||
page.wait_for_selector("table tbody tr", timeout=3000)
|
||||
page.wait_for_selector(
|
||||
"table tbody tr",
|
||||
timeout=TABLE_TR_TIMEOUT_MS,
|
||||
)
|
||||
found_table = True
|
||||
break
|
||||
except Exception:
|
||||
logger.info(
|
||||
"platform_collector: URL=%s 等待表格加载 (attempt %d/20)...",
|
||||
cur_url, attempt + 1,
|
||||
"platform_collector: URL=%s 等待表格加载 (attempt %d/%d)...",
|
||||
cur_url, attempt + 1, TABLE_POLL_MAX_ATTEMPTS,
|
||||
)
|
||||
|
||||
if not found_table:
|
||||
_dump_page(page, download_dir.parent / "sandbox_dump.html", "sandbox")
|
||||
logger.warning("platform_collector: 40s 后未找到表格")
|
||||
return _error_result("table_not_found_after_40s", 0)
|
||||
logger.warning("platform_collector: 表格未找到")
|
||||
raise TableNotFoundError(
|
||||
f"{TABLE_POLL_MAX_ATTEMPTS * TABLE_POLL_INTERVAL_MS // 1000}s "
|
||||
f"内未找到表格 (URL={page.url})"
|
||||
)
|
||||
|
||||
# 表格已找到,等 tbody 完全填充 + dump 一份完整 DOM(用于诊断按钮 selector)
|
||||
try:
|
||||
page.wait_for_function(
|
||||
"document.querySelectorAll('table tbody tr').length > 0",
|
||||
timeout=10_000,
|
||||
timeout=DEFAULT_PAGE_TIMEOUT_MS,
|
||||
)
|
||||
page.wait_for_timeout(1500)
|
||||
page.wait_for_timeout(FULL_RENDER_WAIT_MS)
|
||||
_dump_page(
|
||||
page,
|
||||
download_dir.parent / "hr_contracts_full_dump.html",
|
||||
@@ -401,8 +567,7 @@ def collect_all(limit: int = 0, headless: bool = True) -> dict:
|
||||
)
|
||||
btn.click()
|
||||
# 等卡片出现(页面下方追加 h2「续聘准备任务书」/「单位清单」)
|
||||
# 修复:1500ms 太短,sandbox mock 渲染卡片动画慢,导致抓到的文本不全
|
||||
page.wait_for_timeout(3000)
|
||||
page.wait_for_timeout(CARD_APPEAR_WAIT_MS)
|
||||
# 抓表格下方所有 card 文本
|
||||
cards_text = page.locator(
|
||||
"main .grid > div, main > div > div > div"
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"""
|
||||
|
||||
SKILL_SLUG = "alert-contract-expiring-hr"
|
||||
SKILL_VERSION = "1.1.5"
|
||||
SKILL_VERSION = "1.1.6"
|
||||
LOG_LOGGER_NAME = "openclaw.skill.alert_contract_expiring_hr"
|
||||
PLATFORM_KIT_MIN_VERSION = "1.2.11"
|
||||
|
||||
@@ -43,3 +43,47 @@ DEFAULT_MATERIALS = [
|
||||
MORAL_FAIL_THRESHOLD = 60 # 师德 < 60 直接 non_renew
|
||||
PERFORMANCE_FAIL_THRESHOLD = 60 # 单年考核 < 60 算不合格
|
||||
PERFORMANCE_RENEW_MIN = 80 # 全部 >= 80 且逐年上升 → renew
|
||||
|
||||
|
||||
# ----- v1.1.6:RPA 超时常量(与 skill-template human_verification.py 对齐)-----
|
||||
# 手工登录等待(HITL fallback)
|
||||
HUMAN_WAIT_TIMEOUT = 180 # 手工登录等待超时(秒)—— skill-template 同名常量
|
||||
HUMAN_POLL_INTERVAL = 2.0 # 轮询间隔(秒)—— skill-template 同名常量
|
||||
|
||||
# Playwright 操作超时(毫秒)
|
||||
DEFAULT_PAGE_TIMEOUT_MS = 15_000 # Playwright 默认操作超时 —— skill-template 同名常量
|
||||
GOTO_TIMEOUT_MS = 60_000 # page.goto 导航超时 —— skill-template 同名常量
|
||||
|
||||
# SPA 登录等待(毫秒)
|
||||
SPA_LOGIN_SETTLE_MS = 4_000 # login click 后 settle 时间(SPA 写 cookie)
|
||||
SPA_LOGIN_INPUT_TIMEOUT_MS = 20_000 # 等 username input 出现
|
||||
SPA_LOGIN_URL_TIMEOUT_MS = 30_000 # 等 URL 跳离 login 页
|
||||
SPA_HYDRATION_MS = 3_000 # goto login URL 后 SPA 框架初始化
|
||||
POST_LOGIN_SETTLE_MS = 2_000 # 登录后等 SPA 路由稳定
|
||||
SPA_HOME_RETRY_DELAY_MS = 2_500 # SPA_HOME retry 间隔
|
||||
SPA_HOME_RETRY_MAX = 2 # SPA_HOME retry 最大次数
|
||||
|
||||
# 表格等待(毫秒)
|
||||
TABLE_INITIAL_WAIT_MS = 3_000 # 表格初始等待(React + mock 填充)
|
||||
TABLE_POLL_INTERVAL_MS = 2_000 # 表格轮询间隔
|
||||
TABLE_POLL_MAX_ATTEMPTS = 20 # 表格轮询最大次数(合计 ~40s)
|
||||
TABLE_TR_TIMEOUT_MS = 3_000 # 单次 wait_for_selector 超时
|
||||
FULL_RENDER_WAIT_MS = 1_500 # 完整 DOM 渲染后等渲染结束
|
||||
|
||||
# 侧边栏/卡片(毫秒)
|
||||
SIDEBAR_LINK_TIMEOUT_MS = 30_000 # 等侧边栏 hr/contracts link
|
||||
PUSHSTATE_SETTLE_MS = 1_500 # SPA pushState 后等
|
||||
SIDEBAR_CLICK_URL_TIMEOUT_MS = 10_000 # click 后等 URL 变 /hr/contracts
|
||||
URL_AFTER_LOGIN_TIMEOUT_MS = 20_000 # 根域 login 后等跳转
|
||||
|
||||
# reload 后备用
|
||||
RELOAD_NETWORKIDLE_TIMEOUT_MS = 30_000 # page.reload networkidle 超时
|
||||
RELOAD_DOMCONTENTLOADED_TIMEOUT_MS = 30_000 # page.reload domcontentloaded 超时
|
||||
POST_RELOAD_LOGIN_HYDRATION_MS = 3_000 # reload 后回到 login 页的 hydration
|
||||
POST_RELOGIN_GOTO_WAIT_MS = 2_000 # 重新登录后 goto PLATFORM_URL 等
|
||||
|
||||
# 卡片
|
||||
CARD_APPEAR_WAIT_MS = 3_000 # 「续聘材料包」click 后等卡片出现
|
||||
|
||||
# dump
|
||||
DUMP_PATH_PARENT_MKDIR = True # _dump_page 是否自动创建父目录
|
||||
|
||||
@@ -558,6 +558,131 @@ def test_html_render_unknown_task_raises() -> tuple[bool, str]:
|
||||
return False, "应抛 ValueError,实际未抛"
|
||||
|
||||
|
||||
# ---------- v1.1.6:service.platform_collector 纯函数 ----------
|
||||
|
||||
def test_gen_contract_id_emp_id() -> tuple[bool, str]:
|
||||
"""_gen_contract_id 工号派生(v1.1.1 起用工号作主键)"""
|
||||
from service import platform_collector as pc
|
||||
|
||||
cases = [
|
||||
(("T20190033", "2026-04-15"), "RPA-T20190033-20260415"),
|
||||
(("T20200055", "2026-06-30"), "RPA-T20200055-20260630"),
|
||||
(("T20180021", "2026-08-31"), "RPA-T20180021-20260831"),
|
||||
]
|
||||
for (emp_id, end_date), expected in cases:
|
||||
got = pc._gen_contract_id(emp_id, end_date)
|
||||
if got != expected:
|
||||
return False, f"({emp_id!r}, {end_date!r}) → {got!r},want {expected!r}"
|
||||
return True, f"3 工号派生正确"
|
||||
|
||||
|
||||
def test_gen_contract_id_special_chars() -> tuple[bool, str]:
|
||||
"""_gen_contract_id 工号含特殊字符时正确清洗(保留前 10 字符)"""
|
||||
from service import platform_collector as pc
|
||||
|
||||
# 工号含非 \w 字符 → 清洗;超过 10 字符截断
|
||||
got = pc._gen_contract_id("T-2019.0033", "2026-04-15")
|
||||
# 清洗后保留 \w,"T-2019.0033" → "T20190033"(8 字符,未截断)
|
||||
if "RPA-" not in got or "20260415" not in got:
|
||||
return False, f"清洗后格式错:{got!r}"
|
||||
# 长工号 → 截断到 10 字符
|
||||
got2 = pc._gen_contract_id("ABCDEFGHIJKLMNO", "2026-01-01")
|
||||
# "ABCDEFGHIJ"(10 字符)
|
||||
if not got2.startswith("RPA-ABCDEFGHIJ-"):
|
||||
return False, f"长工号未截断到 10 字符:{got2!r}"
|
||||
return True, "清洗 + 截断 OK"
|
||||
|
||||
|
||||
def test_normalize_date_iso() -> tuple[bool, str]:
|
||||
"""_normalize_date ISO 日期直通"""
|
||||
from service import platform_collector as pc
|
||||
|
||||
cases = [
|
||||
("2026-04-15", "2026-04-15"),
|
||||
("2024-01-01", "2024-01-01"),
|
||||
]
|
||||
for inp, want in cases:
|
||||
got = pc._normalize_date(inp)
|
||||
if got != want:
|
||||
return False, f"{inp!r} → {got!r},want {want!r}"
|
||||
return True, "ISO 日期直通"
|
||||
|
||||
|
||||
def test_normalize_date_fallback() -> tuple[bool, str]:
|
||||
"""_normalize_date 非标准格式截前 10 字符"""
|
||||
from service import platform_collector as pc
|
||||
|
||||
# 非标准但带 ISO 前缀
|
||||
got = pc._normalize_date("2026-04-15T00:00:00")
|
||||
if got != "2026-04-15":
|
||||
return False, f"ISO 前缀应保留前 10 字符 → {got!r}"
|
||||
# 完全非日期 → 截前 10 字符
|
||||
got2 = pc._normalize_date("20260415X")
|
||||
if got2 != "20260415X": # 10 字符
|
||||
return False, f"非日期应截前 10 字符 → {got2!r}"
|
||||
# 带空白
|
||||
got3 = pc._normalize_date(" 2026-04-15 ")
|
||||
if got3 != "2026-04-15":
|
||||
return False, f"带空白应去除 → {got3!r}"
|
||||
return True, "非标准格式 fallback OK"
|
||||
|
||||
|
||||
def test_parse_days_positive() -> tuple[bool, str]:
|
||||
"""_parse_days 正天数"""
|
||||
from service import platform_collector as pc
|
||||
|
||||
cases = [
|
||||
("30", 30),
|
||||
("90", 90),
|
||||
("还剩 45 天", 45),
|
||||
("0", 0),
|
||||
]
|
||||
for inp, want in cases:
|
||||
got = pc._parse_days(inp)
|
||||
if got != want:
|
||||
return False, f"{inp!r} → {got!r},want {want!r}"
|
||||
return True, "正天数解析 OK"
|
||||
|
||||
|
||||
def test_parse_days_negative() -> tuple[bool, str]:
|
||||
"""_parse_days 负数 / 兜底"""
|
||||
from service import platform_collector as pc
|
||||
|
||||
# 含负号
|
||||
if pc._parse_days("-15") != -15:
|
||||
return False, "负天数未解析"
|
||||
# 已过期(带 "已过期" 文本)
|
||||
if pc._parse_days("已过期 7 天") != 7:
|
||||
return False, "已过期文本未解析数字部分"
|
||||
# 无法解析
|
||||
if pc._parse_days("") is not None:
|
||||
return False, "空字符串应返回 None"
|
||||
if pc._parse_days("无数字") is not None:
|
||||
return False, "纯文本应返回 None"
|
||||
return True, "负数 + 兜底 OK"
|
||||
|
||||
|
||||
def test_parse_alert_red_orange_yellow_unknown() -> tuple[bool, str]:
|
||||
"""_parse_alert 红/橙/黄/未知四级映射"""
|
||||
from service import platform_collector as pc
|
||||
|
||||
cases = [
|
||||
("红色预警", "red"),
|
||||
("红牌", "red"),
|
||||
("橙色预警", "orange"),
|
||||
("黄灯", "yellow"),
|
||||
("黄色预警", "yellow"),
|
||||
("无", "unknown"),
|
||||
("", "unknown"),
|
||||
("blue", "unknown"),
|
||||
]
|
||||
for inp, want in cases:
|
||||
got = pc._parse_alert(inp)
|
||||
if got != want:
|
||||
return False, f"{inp!r} → {got!r},want {want!r}"
|
||||
return True, "四级映射 OK"
|
||||
|
||||
|
||||
# ---------- 入口 ----------
|
||||
|
||||
def main() -> int:
|
||||
@@ -592,6 +717,13 @@ def main() -> int:
|
||||
_run("scanner/skips_non_active", test_scanner_skips_non_active_contracts)
|
||||
_run("html_renderer/basic", test_html_render_basic)
|
||||
_run("html_renderer/unknown_task_raises", test_html_render_unknown_task_raises)
|
||||
_run("platform_collector/gen_contract_id_emp_id", test_gen_contract_id_emp_id)
|
||||
_run("platform_collector/gen_contract_id_special", test_gen_contract_id_special_chars)
|
||||
_run("platform_collector/normalize_date_iso", test_normalize_date_iso)
|
||||
_run("platform_collector/normalize_date_fallback", test_normalize_date_fallback)
|
||||
_run("platform_collector/parse_days_positive", test_parse_days_positive)
|
||||
_run("platform_collector/parse_days_negative", test_parse_days_negative)
|
||||
_run("platform_collector/parse_alert_levels", test_parse_alert_red_orange_yellow_unknown)
|
||||
|
||||
total = len(_RESULTS)
|
||||
passed = sum(1 for _, ok, _ in _RESULTS if ok)
|
||||
|
||||
Reference in New Issue
Block a user