278 lines
9.4 KiB
Python
278 lines
9.4 KiB
Python
"""Playwright RPA：从 SmartCampus HR 平台抓合同表 + 续聘材料包。
|
||
|
||
设计要点（v1.1.6，按 skill-template examples/real_browser_rpa 设计模式）：
|
||
- 使用 launch_persistent_context 复用浏览器登录态，避免密码硬编码
|
||
- 首次使用需用户手工登录一次（CLI 默认 --headless false 便于交互）
|
||
- 后续使用无头模式（profile 已存 cookie）
|
||
- **登录策略**：自动 fill demo/123456（sandbox 已知凭据） + 失败时 fallback 手工登录等待（轮询 180s）
|
||
- 抓表格：DOM eval_on_selector_all → JSON
|
||
- 抓材料：点 "续聘材料包" 按钮 → 抓卡片 inner_text
|
||
|
||
依赖为可选：playwright 未安装时 import 抛错，上层 CLI 捕获并降级报错。
|
||
|
||
导航流程（v1.1.1 强化版 — 强制 goto）：
|
||
1. goto 根域
|
||
2. 根域 login（如仍显示表单）→ 等跳到 /home 或 /select
|
||
3. 强制 goto sandbox SPA login URL
|
||
4. SPA login：自动 fill demo/123456，失败 fallback 手工等待（HUMAN_WAIT_TIMEOUT=180s）
|
||
5. 强制 goto SPA_HOME
|
||
6. click 侧边栏 hr/contracts
|
||
7. 等 table tbody tr（最长 ~40s）
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import re
|
||
import sqlite3
|
||
import time
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
|
||
from db import repository as repo
|
||
from util.constants import (
|
||
CARD_APPEAR_WAIT_MS,
|
||
DEFAULT_PAGE_TIMEOUT_MS,
|
||
FULL_RENDER_WAIT_MS,
|
||
GOTO_TIMEOUT_MS,
|
||
HUMAN_POLL_INTERVAL,
|
||
HUMAN_PROMPT_INTERVAL,
|
||
HUMAN_WAIT_TIMEOUT,
|
||
POST_LOGIN_SETTLE_MS,
|
||
POST_RELOAD_LOGIN_HYDRATION_MS,
|
||
POST_RELOGIN_GOTO_WAIT_MS,
|
||
PUSHSTATE_SETTLE_MS,
|
||
RELOAD_DOMCONTENTLOADED_TIMEOUT_MS,
|
||
RELOAD_NETWORKIDLE_TIMEOUT_MS,
|
||
SIDEBAR_CLICK_URL_TIMEOUT_MS,
|
||
SIDEBAR_LINK_TIMEOUT_MS,
|
||
SPA_HOME_RETRY_DELAY_MS,
|
||
SPA_HOME_RETRY_MAX,
|
||
SPA_HYDRATION_MS,
|
||
SPA_LOGIN_INPUT_TIMEOUT_MS,
|
||
SPA_LOGIN_SETTLE_MS,
|
||
SPA_LOGIN_URL_TIMEOUT_MS,
|
||
TABLE_INITIAL_WAIT_MS,
|
||
TABLE_POLL_INTERVAL_MS,
|
||
TABLE_POLL_MAX_ATTEMPTS,
|
||
TABLE_TR_TIMEOUT_MS,
|
||
URL_STABLE_POLL_S,
|
||
URL_STABLE_REQUIRED_S,
|
||
URL_AFTER_LOGIN_TIMEOUT_MS,
|
||
)
|
||
from util.logging_setup import get_skill_logger
|
||
from util.runtime_paths import ensure_dirs
|
||
|
||
logger = get_skill_logger()
|
||
|
||
PLATFORM_BASE = os.environ.get("SMARTCAMPUS_BASE_URL", "https://sandbox.jc2009.com")
|
||
PLATFORM_HR_PATH = os.environ.get(
|
||
"SMARTCAMPUS_HR_PATH", "/industries/campus/smartcampus/app/hr/contracts"
|
||
)
|
||
PLATFORM_URL = PLATFORM_BASE + PLATFORM_HR_PATH
|
||
|
||
ROOT_URL = PLATFORM_BASE + "/"
|
||
SELECT_URL = PLATFORM_BASE + "/select"
|
||
SPA_HOME_URL = PLATFORM_BASE + "/industries/campus/smartcampus/app/home"
|
||
SPA_LOGIN_URL = PLATFORM_BASE + "/industries/campus/smartcampus/login"
|
||
|
||
|
||
# ---------- 错误码（与 skill-template 对齐） ----------
|
||
|
||
class RPAError(Exception):
|
||
"""RPA 流程错误基类。"""
|
||
|
||
code: str = "RPA_ERROR"
|
||
|
||
def __init__(self, message: str):
|
||
super().__init__(message)
|
||
self.message = message
|
||
|
||
|
||
class LoginTimeoutError(RPAError):
|
||
"""手工登录超时（HUMAN_WAIT_TIMEOUT 秒内未检测到登录成功）。
|
||
|
||
错误码：ERROR:LOGIN_TIMEOUT（与 skill-template task_rpa.py 对齐）
|
||
"""
|
||
|
||
code = "ERROR:LOGIN_TIMEOUT"
|
||
|
||
|
||
class MissingBrowserError(RPAError):
|
||
"""未检测到 Chrome/Edge。
|
||
|
||
错误码：ERROR:MISSING_BROWSER（与 skill-template browser_session.py 对齐）
|
||
"""
|
||
|
||
code = "ERROR:MISSING_BROWSER"
|
||
|
||
|
||
class TableNotFoundError(RPAError):
|
||
"""40s 内未找到表格。
|
||
|
||
错误码：ERROR:TABLE_NOT_FOUND
|
||
"""
|
||
|
||
code = "ERROR:TABLE_NOT_FOUND"
|
||
|
||
|
||
# ---------- 工具函数 ----------
|
||
|
||
def _safe_getenv_dir(env_key: str, default_rel: str) -> Path:
|
||
raw = os.environ.get(env_key)
|
||
if raw:
|
||
return Path(raw).resolve()
|
||
from util.runtime_paths import get_data_dir
|
||
|
||
return (get_data_dir() / default_rel).resolve()
|
||
|
||
|
||
def _get_env(env_key: str, default: str = "") -> str:
|
||
"""v1.1.31/32：读 env 值，进程 env 优先，缺则从 .env 文件读（手动解析）。
|
||
|
||
v1.1.32 关键修复：之前只读 os.environ，但 .env 是用户手动维护的文件，
|
||
python 进程默认不会自动加载它，导致 SMARTCAMPUS_LOGIN_USER/PASSWORD 永远读不到
|
||
（jiangchang 宿主未必会注入 .env）。手动解析 .env 兜底。
|
||
"""
|
||
# 1. 进程 env 优先（jiangchang 注入或系统 env 优先）
|
||
val = os.environ.get(env_key)
|
||
if val is not None and val != "":
|
||
return val
|
||
|
||
# 2. 从 .env 文件手动解析（一次性缓存）
|
||
global _ENV_CACHE_LOADED, _ENV_CACHE
|
||
if not _ENV_CACHE_LOADED:
|
||
_ENV_CACHE_LOADED = True
|
||
_ENV_CACHE = _load_env_file()
|
||
return _ENV_CACHE.get(env_key, default)
|
||
|
||
|
||
_ENV_CACHE: dict[str, str] = {}
|
||
_ENV_CACHE_LOADED = False
|
||
|
||
|
||
def _load_env_file() -> dict[str, str]:
|
||
"""手动解析 .env 文件（不依赖 python-dotenv）。
|
||
|
||
规则：
|
||
- 跳过注释（# 开头）和空行
|
||
- 形如 `KEY=value`，value 可带 "..." 或 '...' 引号（自动去引号）
|
||
- value 含 `=` 时只 split 一次
|
||
- 找不到 .env 返回空 dict
|
||
"""
|
||
from util.runtime_paths import get_env_path
|
||
|
||
env_path = get_env_path()
|
||
if not env_path.exists():
|
||
return {}
|
||
out: dict[str, str] = {}
|
||
try:
|
||
for raw in env_path.read_text(encoding="utf-8").splitlines():
|
||
line = raw.strip()
|
||
if not line or line.startswith("#"):
|
||
continue
|
||
if "=" not in line:
|
||
continue
|
||
k, v = line.split("=", 1)
|
||
k = k.strip()
|
||
v = v.strip()
|
||
# 去引号
|
||
if len(v) >= 2 and v[0] == v[-1] and v[0] in ("'", '"'):
|
||
v = v[1:-1]
|
||
out[k] = v
|
||
except Exception as e:
|
||
logger.warning("platform_collector: 解析 .env 失败（继续走空 dict）: %s", e)
|
||
return out
|
||
|
||
|
||
def _goto(page, url: str, timeout: int = GOTO_TIMEOUT_MS) -> None:
|
||
"""带 networkidle / domcontentloaded 双兜底的 goto。"""
|
||
try:
|
||
page.goto(url, wait_until="networkidle", timeout=timeout)
|
||
except Exception as e:
|
||
logger.warning("goto networkidle 失败 %s: %s；尝试 domcontentloaded", url, e)
|
||
try:
|
||
page.goto(url, wait_until="domcontentloaded", timeout=timeout)
|
||
except Exception as e2:
|
||
logger.warning("goto domcontentloaded 也失败 %s: %s", url, e2)
|
||
|
||
|
||
def _dump_page(page, path: Path, label: str) -> None:
|
||
"""Dump 当前页面 HTML 到指定路径，自动创建父目录。修复：父目录可能不存在导致静默失败。"""
|
||
try:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
html = page.content()
|
||
path.write_text(html, encoding="utf-8")
|
||
logger.info(
|
||
"platform_collector: %s dump → %s (%d 字节)",
|
||
label, path, len(html),
|
||
)
|
||
except Exception as e:
|
||
logger.warning("dump %s 失败: %s", label, e)
|
||
|
||
|
||
def _is_logged_in(page) -> bool:
|
||
"""检测 sandbox SPA 是否已登录（URL 含 /app/ 且不含 /login 或 /signin）。
|
||
|
||
与 skill-template real_browser_rpa _is_logged_in 概念对齐（这里只看 URL，
|
||
因为 sandbox SPA 登录后 URL 稳定跳到 /app/...）。
|
||
"""
|
||
url_lower = page.url.lower()
|
||
return (
|
||
"/app/" in url_lower
|
||
and "/login" not in url_lower
|
||
and "/signin" not in url_lower
|
||
)
|
||
|
||
|
||
def _is_spa_routed(page) -> bool:
|
||
"""v1.1.22：检测 sandbox SPA 是否已路由到 SPA 页面（含 /select /app/）。"""
|
||
url_lower = page.url.lower()
|
||
return (
|
||
("/app/" in url_lower or "/select" in url_lower)
|
||
and "/login" not in url_lower
|
||
and "/signin" not in url_lower
|
||
)
|
||
|
||
|
||
# ---------- v1.1.31：从 service.platform_collector_login re-export 登录流程函数 ----------
|
||
# facade 仅做 re-export，不持有实现；测试 / CLI 通过 service.platform_collector.X 仍可访问。
|
||
from service.platform_collector_login import ( # noqa: E402,F401
|
||
_try_auto_sign_in as _try_auto_sign_in,
|
||
_wait_for_login_inputs_filled as _wait_for_login_inputs_filled,
|
||
_is_login_page_url_for_user as _is_login_page_url_for_user,
|
||
_wait_for_manual_login as _wait_for_manual_login,
|
||
_wait_for_url_stable as _wait_for_url_stable,
|
||
_fill_root_login as _fill_root_login,
|
||
_fill_spa_login_from_env as _fill_spa_login_from_env,
|
||
_is_dev_mode_panel_open as _is_dev_mode_panel_open,
|
||
_resolve_target_industry as _resolve_target_industry,
|
||
_click_dev_mode_sign_in_for_industry as _click_dev_mode_sign_in_for_industry,
|
||
_click_campus_link as _click_campus_link,
|
||
_fill_spa_login as _fill_spa_login,
|
||
)
|
||
|
||
# ---------- v1.1.31：从 service.platform_collector_collect re-export 采集主流程 ----------
|
||
from service.platform_collector_collect import ( # noqa: E402,F401
|
||
collect_all as collect_all,
|
||
_error_result as _error_result,
|
||
)
|
||
|
||
# ---------- v1.1.31：从 service.platform_collector_persist re-export DB 持久化 ----------
|
||
from service.platform_collector_persist import ( # noqa: E402,F401
|
||
_persist_contracts as _persist_contracts,
|
||
_enqueue_download_tasks_safe as _enqueue_download_tasks_safe,
|
||
)
|
||
|
||
|
||
# ---------- v1.1.31：从 service.platform_collector_queue re-export 下载队列 + 解析 helpers ----------
|
||
from service.platform_collector_queue import ( # noqa: E402,F401
|
||
_download_materials_for_row as _download_materials_for_row,
|
||
_find_row_for_contract as _find_row_for_contract,
|
||
process_queue as process_queue,
|
||
_gen_contract_id as _gen_contract_id,
|
||
_normalize_date as _normalize_date,
|
||
_parse_days as _parse_days,
|
||
_parse_alert as _parse_alert,
|
||
)
|