334 lines
13 KiB
Python
334 lines
13 KiB
Python
"""Playwright RPA：SmartCampus HR 平台下载队列处理 + 合同 ID 解析 helpers。
|
||
|
||
v1.1.31：从 platform_collector.py 拆分出来的队列处理子模块。
|
||
包含 7 个函数：
|
||
- _download_materials_for_row（点续聘材料包按钮抓卡片文本）
|
||
- _find_row_for_contract（在 hr/contracts 表格找 contract_id 行号）
|
||
- process_queue（领取 download_tasks 队列、登录、抓材料、落库）
|
||
- _gen_contract_id / _normalize_date / _parse_days / _parse_alert（4 个纯解析 helper）
|
||
|
||
设计要点（保持 v1.1.30 不变）：
|
||
- 登录子流程通过 service.platform_collector_login 顶部 import
|
||
- facade 持有的符号（_goto / _is_logged_in / 异常类 / SPA_LOGIN_URL 等）
|
||
通过 `from service import platform_collector as _pc` lazy import
|
||
- 4 个 helper 函数（_gen_contract_id / _normalize_date / _parse_days / _parse_alert）
|
||
在本模块内自包含，collect / persist / queue 都可能用到（通过 facade re-export 访问）
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
import sqlite3
|
||
from datetime import datetime
|
||
|
||
from db import repository as repo
|
||
from db.connection import get_connection
|
||
from db.migrations import ensure_materials_content_schema
|
||
from util.constants import (
|
||
CARD_APPEAR_WAIT_MS,
|
||
DEFAULT_PAGE_TIMEOUT_MS,
|
||
HUMAN_WAIT_TIMEOUT,
|
||
POST_LOGIN_SETTLE_MS,
|
||
SPA_HOME_RETRY_DELAY_MS,
|
||
SPA_HOME_RETRY_MAX,
|
||
SPA_HYDRATION_MS,
|
||
TABLE_INITIAL_WAIT_MS,
|
||
TABLE_POLL_INTERVAL_MS,
|
||
TABLE_POLL_MAX_ATTEMPTS,
|
||
TABLE_TR_TIMEOUT_MS,
|
||
)
|
||
from util.logging_setup import get_skill_logger
|
||
from service.platform_collector_login import (
|
||
_fill_spa_login,
|
||
_wait_for_manual_login,
|
||
_wait_for_url_stable,
|
||
)
|
||
|
||
logger = get_skill_logger()
|
||
|
||
|
||
def _download_materials_for_row(
|
||
page, row_idx: int, contract_id: str, mat_btn_count: int,
|
||
) -> list[dict]:
|
||
"""点续聘材料包按钮抓卡片文本（与 v1.1.17 collect_all 内联逻辑等价）。
|
||
|
||
返回 materials 列表（kind=sandbox_card，含 file_name + text）。
|
||
"""
|
||
if mat_btn_count == 0:
|
||
return []
|
||
materials: list[dict] = []
|
||
try:
|
||
btn = page.locator("table tbody tr").nth(row_idx).locator(
|
||
"button:has-text('续聘材料包')"
|
||
)
|
||
btn.click()
|
||
page.wait_for_timeout(CARD_APPEAR_WAIT_MS)
|
||
cards_text = page.locator(
|
||
"main .grid > div, main > div > div > div"
|
||
).all_inner_texts()
|
||
materials_text = "\n\n".join(t.strip() for t in cards_text if t.strip())
|
||
if materials_text:
|
||
materials.append({
|
||
"kind": "sandbox_card",
|
||
"file_name": f"{contract_id}_materials.txt",
|
||
"text": materials_text,
|
||
})
|
||
logger.info(
|
||
"platform_collector: 抓到卡片文本 %d 字 (%s)",
|
||
len(materials_text), contract_id,
|
||
)
|
||
except Exception as e:
|
||
logger.warning("抓卡片失败 %s: %s", contract_id, e)
|
||
return materials
|
||
|
||
|
||
def _find_row_for_contract(page, contract_id: str) -> int | None:
|
||
"""在 hr/contracts 表格中找 contract_id 所在行号。
|
||
|
||
支持 7 列（用 teacher_name 派生 contract_id）与 10 列（用 emp_id）。
|
||
不匹配返回 None（调用方 mark_task_failed）。
|
||
"""
|
||
rows = page.eval_on_selector_all(
|
||
"table tbody tr",
|
||
"rs => rs.map(r => Array.from(r.querySelectorAll('td')).map(td => td.innerText))",
|
||
)
|
||
for i, r in enumerate(rows):
|
||
ncols = len(r)
|
||
if ncols < 6:
|
||
continue
|
||
first_col = r[0].strip()
|
||
is_emp_id = bool(re.match(r"^[A-Za-z]?\d+$", first_col))
|
||
if is_emp_id and ncols >= 9:
|
||
emp_id, contract_end = first_col, r[5].split("~")[-1].strip()
|
||
elif ncols >= 6:
|
||
emp_id, contract_end = first_col, r[2].strip()
|
||
else:
|
||
continue
|
||
candidate = _gen_contract_id(emp_id, contract_end)
|
||
if candidate == contract_id:
|
||
return i
|
||
return None
|
||
|
||
|
||
def process_queue(
|
||
limit: int = 5,
|
||
headless: bool = True,
|
||
human_wait_timeout: int = HUMAN_WAIT_TIMEOUT,
|
||
ids_filter: str = "",
|
||
) -> dict:
|
||
"""从 download_tasks 队列取 pending 任务，对每个合同重新走材料抓取。
|
||
|
||
与 collect_all 重复登录流程，但每行只点「续聘材料包」按钮抓卡片，不重新抓合同表。
|
||
|
||
参数：
|
||
limit: 本次最多处理任务数（0 = 不限）
|
||
ids_filter: 逗号分隔 task_id 列表（只处理这些 pending）
|
||
headless: 无头模式（需要 profile cookie 已保存）
|
||
human_wait_timeout: 手工登录 fallback 等待秒数
|
||
|
||
返回 {processed, succeeded, failed, details}。
|
||
"""
|
||
# Lazy import: 读 facade 持有的 _safe_getenv_dir / _goto / _is_logged_in / 异常类 / URL 常量
|
||
from service import platform_collector as _pc
|
||
|
||
try:
|
||
from playwright.sync_api import sync_playwright
|
||
except ImportError:
|
||
logger.error("playwright 未安装，跳过队列处理")
|
||
return {"processed": 0, "succeeded": 0, "failed": 0, "details": [],
|
||
"error": "playwright_not_installed"}
|
||
|
||
# 1. 领取一批任务
|
||
ids_list = [s.strip() for s in ids_filter.split(",") if s.strip()] if ids_filter else None
|
||
conn = get_connection()
|
||
try:
|
||
tasks = repo.claim_pending_tasks(
|
||
conn,
|
||
limit=limit if limit > 0 else 999,
|
||
ids_filter=ids_list,
|
||
)
|
||
finally:
|
||
conn.close()
|
||
|
||
if not tasks:
|
||
return {"processed": 0, "succeeded": 0, "failed": 0, "details": []}
|
||
|
||
profile_dir = _pc._safe_getenv_dir("SMARTCAMPUS_PROFILE_DIR", "playwright-profile")
|
||
download_dir = _pc._safe_getenv_dir("MATERIALS_DOWNLOAD_DIR", "materials")
|
||
profile_dir.mkdir(parents=True, exist_ok=True)
|
||
download_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
details: list[dict] = []
|
||
succeeded = 0
|
||
failed = 0
|
||
|
||
with sync_playwright() as p:
|
||
ctx = p.chromium.launch_persistent_context(
|
||
user_data_dir=str(profile_dir),
|
||
headless=headless,
|
||
accept_downloads=True,
|
||
viewport={"width": 1440, "height": 900},
|
||
channel="msedge" if not headless else None,
|
||
) if not headless else p.chromium.launch_persistent_context(
|
||
user_data_dir=str(profile_dir),
|
||
headless=True,
|
||
accept_downloads=True,
|
||
viewport={"width": 1440, "height": 900},
|
||
)
|
||
try:
|
||
page = ctx.new_page()
|
||
page.set_default_timeout(DEFAULT_PAGE_TIMEOUT_MS)
|
||
|
||
# 登录：复用 v1.1.17 collect_all 的探针 + 自动 fill + 手工 fallback 流程
|
||
logger.info("platform_collector.process_queue: 启动 profile=%s headless=%s 待处理 %d",
|
||
profile_dir, headless, len(tasks))
|
||
_pc._goto(page, _pc.SPA_HOME_URL)
|
||
page.wait_for_timeout(SPA_HYDRATION_MS)
|
||
if _pc._is_logged_in(page):
|
||
logger.info("platform_collector.process_queue: ✅ 已登录态")
|
||
else:
|
||
_pc._goto(page, _pc.SPA_LOGIN_URL)
|
||
page.wait_for_timeout(SPA_HYDRATION_MS)
|
||
if not _fill_spa_login(page):
|
||
if not headless:
|
||
_wait_for_manual_login(page, wait_sec=human_wait_timeout)
|
||
else:
|
||
raise _pc.LoginTimeoutError("process_queue headless 自动 fill 失败")
|
||
else:
|
||
page.wait_for_timeout(POST_LOGIN_SETTLE_MS)
|
||
|
||
# 强制 goto SPA_HOME + 侧边栏 hr/contracts
|
||
for _ in range(SPA_HOME_RETRY_MAX):
|
||
if _pc._is_logged_in(page):
|
||
break
|
||
_pc._goto(page, _pc.SPA_HOME_URL)
|
||
page.wait_for_timeout(SPA_HOME_RETRY_DELAY_MS)
|
||
if _fill_spa_login(page):
|
||
page.wait_for_timeout(POST_LOGIN_SETTLE_MS)
|
||
if not _pc._is_logged_in(page):
|
||
raise _pc.LoginTimeoutError("process_queue 仍未登录")
|
||
|
||
# 等表格
|
||
page.wait_for_timeout(TABLE_INITIAL_WAIT_MS)
|
||
found_table = False
|
||
for _attempt in range(TABLE_POLL_MAX_ATTEMPTS):
|
||
page.wait_for_timeout(TABLE_POLL_INTERVAL_MS)
|
||
try:
|
||
page.wait_for_selector(
|
||
"table tbody tr", timeout=TABLE_TR_TIMEOUT_MS,
|
||
)
|
||
found_table = True
|
||
break
|
||
except Exception:
|
||
pass
|
||
if not found_table:
|
||
raise _pc.TableNotFoundError("process_queue 表格未找到")
|
||
|
||
mat_btn_count = page.locator(
|
||
"table tbody tr button:has-text('续聘材料包')"
|
||
).count()
|
||
|
||
# 2. 逐任务处理
|
||
for t in tasks:
|
||
contract_id = t["contract_id"]
|
||
task_id = t["task_id"]
|
||
conn = get_connection()
|
||
try:
|
||
row_idx = _find_row_for_contract(page, contract_id)
|
||
if row_idx is None:
|
||
repo.mark_task_failed(conn, task_id, "contract_not_found_in_table")
|
||
failed += 1
|
||
details.append({"task_id": task_id, "contract_id": contract_id,
|
||
"error": "contract_not_found_in_table"})
|
||
conn.commit()
|
||
continue
|
||
materials = _download_materials_for_row(
|
||
page, row_idx, contract_id, mat_btn_count,
|
||
)
|
||
ensure_materials_content_schema(conn)
|
||
for m in materials:
|
||
if isinstance(m, dict) and m.get("kind") == "sandbox_card":
|
||
conn.execute(
|
||
"""
|
||
INSERT OR REPLACE INTO materials_content
|
||
(contract_id, file_name, file_path, extracted_text,
|
||
parse_method, parsed_at)
|
||
VALUES (?, ?, ?, ?, 'sandbox_card', ?)
|
||
""",
|
||
(
|
||
contract_id, m["file_name"],
|
||
f"sandbox://hr/contracts/{contract_id}",
|
||
m["text"],
|
||
datetime.now().isoformat(timespec="seconds"),
|
||
),
|
||
)
|
||
repo.parse_sandbox_card_performance(conn, contract_id, m["text"])
|
||
result_path = (
|
||
f"sandbox://hr/contracts/{contract_id}"
|
||
if materials else "no_materials_captured"
|
||
)
|
||
repo.mark_task_success(conn, task_id, result_path)
|
||
succeeded += 1
|
||
details.append({
|
||
"task_id": task_id,
|
||
"contract_id": contract_id,
|
||
"ok": True,
|
||
"materials_count": len(materials),
|
||
})
|
||
conn.commit()
|
||
except Exception as e:
|
||
repo.mark_task_failed(conn, task_id, str(e))
|
||
failed += 1
|
||
details.append({
|
||
"task_id": task_id, "contract_id": contract_id,
|
||
"error": str(e),
|
||
})
|
||
conn.commit()
|
||
logger.warning("process_queue: task=%s 失败: %s", task_id, e)
|
||
finally:
|
||
conn.close()
|
||
finally:
|
||
ctx.close()
|
||
|
||
return {
|
||
"processed": len(tasks),
|
||
"succeeded": succeeded,
|
||
"failed": failed,
|
||
"details": details,
|
||
}
|
||
|
||
|
||
def _gen_contract_id(emp_id: str, end_date: str) -> str:
|
||
"""从 (工号, 到期日) 派 contract_id，确保幂等。
|
||
|
||
sandbox v1.1.1 起改用「工号」作主键（之前用姓名）。
|
||
"""
|
||
safe_emp = re.sub(r"[^\w]", "", emp_id)[:10] or "T"
|
||
safe_end = end_date.replace("-", "")[:8]
|
||
return f"RPA-{safe_emp}-{safe_end}"
|
||
|
||
|
||
def _normalize_date(s: str) -> str:
|
||
"""尽量把 sandbox 返回的日期归一为 YYYY-MM-DD。"""
|
||
s = s.strip()
|
||
if re.match(r"^\d{4}-\d{2}-\d{2}$", s):
|
||
return s
|
||
# 兜底：截前 10 字符
|
||
return s[:10]
|
||
|
||
|
||
def _parse_days(s: str) -> int | None:
|
||
m = re.search(r"-?\d+", s)
|
||
return int(m.group(0)) if m else None
|
||
|
||
|
||
def _parse_alert(s: str) -> str:
|
||
s = s.strip()
|
||
if "红" in s:
|
||
return "red"
|
||
if "橙" in s:
|
||
return "orange"
|
||
if "黄" in s:
|
||
return "yellow"
|
||
return "unknown"
|