v1.1.2: sandbox 卡卡片文本采集 + 解析为 performance 记录
sandbox 行为新发现：
1. 「续聘材料包」按钮不是下载文件，而是触发页面下方显示「续聘准备任务书」
+「单位清单」卡片（含 checklist + 近三年考核表）
2. 进入 hr/contracts 页面后需要刷新（page.reload）才能看到表格行
3. 表格列序扩展为 10 列：工号 / 姓名 / 单位 / 岗位 / 合同类型 / 起止日期 / 剩余天数 / 预警 / 建议 / 材料包按钮
4. contract_id 改用工号（稳定，不随姓名变化）
改动：
- platform_collector.collect_all：
* 进入 hr/contracts 后 page.reload 刷新触发 SPA mock 数据加载
* 新列序映射：r[0]=工号 r[1]=姓名 r[2]=单位 r[3]=岗位 r[4]=合同类型
r[5]=起止日期 r[6]=剩余天数 r[7]=预警 r[8]=建议
* 合同 end_date 从「2024-03-01 ~ 2026-04-15」截取 ~ 后的部分
* 点「续聘材料包」按钮 → 抓下方卡片 inner_text（含 checklist + 考核等次）
* 把卡片文本存入 materials_content（parse_method='sandbox_card'）
- _persist_contracts 增加 sandbox_card 路径分支：写 materials_content 后同步调
repo.parse_sandbox_card_performance → 抽 year + 等次 → 写 performance_records
- repository.parse_sandbox_card_performance：双模式正则匹配
* 行格式 "2024\t称职"
* 叙述式 "2024年：等次「称职」"
- repository.upsert_performance_record + 等次 → 分数映射
（优秀=90 称职=75 基本称职=60 不称职=30）
实测（sandbox 3 合同）：
- RPA-T20190033-20260415 钱实验 → 解析 4 条 performance
- RPA-T20200055-20260630 孙助理 → 解析 6 条
- RPA-T20180021-20260831 王明远 → 解析 6 条
- generate-renewal-task 全部输出 observe + decision_source=rules
Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
@@ -59,6 +59,82 @@ def list_performance_by_contract(
|
||||
).fetchall()
|
||||
|
||||
|
||||
# 等次 → 分数映射（v1.1.1 sandbox card 解析）
|
||||
_GRADE_TO_SCORE = {
|
||||
"优秀": 90.0,
|
||||
"称职": 75.0,
|
||||
"基本称职": 60.0,
|
||||
"不称职": 30.0,
|
||||
"不合格": 30.0,
|
||||
}
|
||||
|
||||
|
||||
def upsert_performance_record(
|
||||
conn: sqlite3.Connection,
|
||||
contract_id: str,
|
||||
year: int,
|
||||
grade: str,
|
||||
) -> bool:
|
||||
"""upsert 一条业绩记录（用 grade 映射分数）。
|
||||
|
||||
sandbox card 文本不含真分数时使用。返回是否实际写入。
|
||||
"""
|
||||
score = _GRADE_TO_SCORE.get(grade)
|
||||
if score is None:
|
||||
return False
|
||||
record_id = f"P-{contract_id}-{year}"
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO performance_records
|
||||
(record_id, contract_id, year, performance_score,
|
||||
moral_teaching_score, research_score, teaching_score)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(record_id) DO UPDATE SET
|
||||
performance_score = excluded.performance_score
|
||||
""",
|
||||
(record_id, contract_id, year, score, score, score, score),
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def parse_sandbox_card_performance(
|
||||
conn: sqlite3.Connection,
|
||||
contract_id: str,
|
||||
card_text: str,
|
||||
) -> int:
|
||||
"""从 sandbox 卡片文本抽 (year, 等次) → upsert 到 performance_records。
|
||||
|
||||
文本片段形如：
|
||||
year result
|
||||
2024 称职
|
||||
2025 优秀
|
||||
2024年：等次「称职」。
|
||||
返回写入条数。
|
||||
"""
|
||||
import re
|
||||
if not card_text:
|
||||
return 0
|
||||
|
||||
written = 0
|
||||
# 模式 1：行格式 "2024\t称职"
|
||||
for m in re.finditer(r"^(\d{4})\s*[ \t]+(优秀|称职|基本称职|不称职|不合格)\s*$",
|
||||
card_text, re.MULTILINE):
|
||||
year = int(m.group(1))
|
||||
grade = m.group(2)
|
||||
if upsert_performance_record(conn, contract_id, year, grade):
|
||||
written += 1
|
||||
|
||||
# 模式 2：叙述式 "2024年：等次「称职」"
|
||||
for m in re.finditer(r"(\d{4})\s*年[：:]\s*等次[「\"']\s*(优秀|称职|基本称职|不称职|不合格)\s*[」\"']",
|
||||
card_text):
|
||||
year = int(m.group(1))
|
||||
grade = m.group(2)
|
||||
if upsert_performance_record(conn, contract_id, year, grade):
|
||||
written += 1
|
||||
|
||||
return written
|
||||
|
||||
|
||||
# ---------- alerts ----------
|
||||
|
||||
def insert_alert(
|
||||
|
||||
@@ -243,6 +243,17 @@ def collect_all(limit: int = 0, headless: bool = True) -> dict:
|
||||
|
||||
logger.info("platform_collector: 当前 URL=%s", page.url)
|
||||
|
||||
# 5.5. 进入页面后刷新（SPA 进入新路由时 mock 数据可能未加载，需刷新触发）
|
||||
try:
|
||||
logger.info("platform_collector: 进入 hr/contracts 后刷新页面")
|
||||
page.reload(wait_until="networkidle", timeout=20_000)
|
||||
except Exception as e:
|
||||
logger.warning("刷新 networkidle 失败: %s；尝试 domcontentloaded", e)
|
||||
try:
|
||||
page.reload(wait_until="domcontentloaded", timeout=20_000)
|
||||
except Exception as e2:
|
||||
logger.warning("刷新 domcontentloaded 也失败: %s", e2)
|
||||
|
||||
# 7. dump 当前 hr/contracts 页面（用于调试）
|
||||
try:
|
||||
html = page.content()
|
||||
@@ -284,6 +295,23 @@ def collect_all(limit: int = 0, headless: bool = True) -> dict:
|
||||
logger.warning("dump 失败: %s", e)
|
||||
return _error_result("table_not_found_after_40s", 0)
|
||||
|
||||
# 表格已找到，等 tbody 完全填充 + dump 一份完整 DOM（用于诊断按钮 selector）
|
||||
try:
|
||||
page.wait_for_function(
|
||||
"document.querySelectorAll('table tbody tr').length > 0",
|
||||
timeout=10_000,
|
||||
)
|
||||
page.wait_for_timeout(1500)
|
||||
html = page.content()
|
||||
full_dump = download_dir.parent / "hr_contracts_full_dump.html"
|
||||
full_dump.write_text(html, encoding="utf-8")
|
||||
logger.info(
|
||||
"platform_collector: 完整 hr/contracts dump → %s (%d 字节)",
|
||||
full_dump, len(html),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("等完整渲染超时: %s", e)
|
||||
|
||||
rows = page.eval_on_selector_all(
|
||||
"table tbody tr",
|
||||
"rs => rs.map(r => Array.from(r.querySelectorAll('td')).map(td => td.innerText))",
|
||||
@@ -302,21 +330,33 @@ def collect_all(limit: int = 0, headless: bool = True) -> dict:
|
||||
except Exception:
|
||||
mat_btn_count = 0
|
||||
|
||||
# 抓「材料包」card 区段的 DOM（用于后续解析每行 click 后的卡片文本）
|
||||
# sandbox 行为：click 按钮后会在表格下方追加「续聘准备任务书」+「单位清单」卡片
|
||||
|
||||
for i, r in enumerate(rows):
|
||||
if limit and i >= limit:
|
||||
break
|
||||
if len(r) < 6:
|
||||
logger.warning("行 %d 列数 < 6，跳过: %s", i, r)
|
||||
if len(r) < 7:
|
||||
logger.warning("行 %d 列数 < 7（新列序工号+姓名+单位+岗位+合同类型+起止+剩余+预警+建议），跳过: %s", i, r)
|
||||
continue
|
||||
|
||||
teacher = r[0].strip()
|
||||
unit = r[1].strip()
|
||||
contract_end = r[2].strip()
|
||||
days_raw = r[3].strip()
|
||||
alert_label = r[4].strip()
|
||||
suggestion_hint = r[5].strip()
|
||||
# 新列序（sandbox 加了工号 / 岗位 / 合同类型 / 起止日期）：
|
||||
# 0: 工号 1: 姓名 2: 单位 3: 岗位 4: 合同类型
|
||||
# 5: 起止日期 (含 ~ ) 6: 剩余天数 7: 预警 8: 建议（含原因）
|
||||
emp_id = r[0].strip()
|
||||
teacher = r[1].strip()
|
||||
unit = r[2].strip()
|
||||
position = r[3].strip()
|
||||
contract_type = r[4].strip()
|
||||
period = r[5].strip()
|
||||
# 起止日期形如 "2024-03-01 ~ 2026-04-15" → 取 ~ 后
|
||||
contract_end = period.split("~")[-1].strip() if "~" in period else period
|
||||
days_raw = r[6].strip()
|
||||
alert_label = r[7].strip()
|
||||
suggestion_hint = r[8].strip() if len(r) > 8 else ""
|
||||
|
||||
contract_id = _gen_contract_id(teacher, contract_end)
|
||||
# 用工号生成 contract_id（稳定，不会随姓名变化）
|
||||
contract_id = _gen_contract_id(emp_id, contract_end)
|
||||
contract = {
|
||||
"contract_id": contract_id,
|
||||
"teacher_name": teacher,
|
||||
@@ -328,32 +368,50 @@ def collect_all(limit: int = 0, headless: bool = True) -> dict:
|
||||
"materials": [],
|
||||
}
|
||||
|
||||
# 只有在按钮存在时才尝试下载（sandbox mock 当前未部署此按钮）
|
||||
# 点「续聘材料包」按钮 → 等卡片展开 → 抓卡片文本（含 checklist）
|
||||
if mat_btn_count > 0:
|
||||
try:
|
||||
btn = page.locator("table tbody tr").nth(i).locator(
|
||||
"button:has-text('续聘材料包')"
|
||||
)
|
||||
with page.expect_download(timeout=10_000) as dl_info:
|
||||
btn.click()
|
||||
dl = dl_info.value
|
||||
fname = dl.suggested_filename or "materials.zip"
|
||||
dest = download_dir / f"{contract_id}_{fname}"
|
||||
dl.save_as(str(dest))
|
||||
contract["materials"].append(str(dest))
|
||||
downloaded += 1
|
||||
btn.click()
|
||||
# 等卡片出现（页面下方追加 h2「续聘准备任务书」/「单位清单」）
|
||||
page.wait_for_timeout(1500)
|
||||
# 抓表格下方所有 card 文本
|
||||
cards_text = page.locator(
|
||||
"main .grid > div, main > div > div > div"
|
||||
).all_inner_texts()
|
||||
materials_text = "\n\n".join(
|
||||
t.strip() for t in cards_text if t.strip()
|
||||
)
|
||||
if materials_text:
|
||||
# 存到 materials_content（sandbox mock 卡片文本作为材料包内容）
|
||||
contract["materials"].append({
|
||||
"kind": "sandbox_card",
|
||||
"file_name": f"{contract_id}_materials.txt",
|
||||
"text": materials_text,
|
||||
})
|
||||
downloaded += 1
|
||||
logger.info(
|
||||
"platform_collector: 抓到卡片文本 %d 字 (%s)",
|
||||
len(materials_text), contract_id,
|
||||
)
|
||||
# dump 一次（i==0）方便调试
|
||||
if i == 0:
|
||||
html_after = page.content()
|
||||
click_dump = download_dir.parent / "after_click_dump.html"
|
||||
click_dump.write_text(html_after, encoding="utf-8")
|
||||
logger.info(
|
||||
"platform_collector: 首行 click 后 dump → %s (%d 字节)",
|
||||
click_dump, len(html_after),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("材料下载失败 %s: %s", contract_id, e)
|
||||
logger.warning("抓卡片失败 %s: %s", contract_id, e)
|
||||
else:
|
||||
logger.info(
|
||||
"platform_collector: sandbox 未部署「续聘材料包」按钮，"
|
||||
"跳过材料下载（仅采集表格列）",
|
||||
)
|
||||
# 仅警告一次
|
||||
if i == 0:
|
||||
logger.warning(
|
||||
"platform_collector: 提示：材料包缺失时，"
|
||||
"parse-materials CLI 将标记为「无材料」"
|
||||
logger.info(
|
||||
"platform_collector: sandbox 未部署「续聘材料包」按钮，"
|
||||
"跳过材料抓取（仅采集表格列）",
|
||||
)
|
||||
|
||||
contracts.append(contract)
|
||||
@@ -413,27 +471,56 @@ def _persist_contracts(contracts: list[dict]) -> int:
|
||||
),
|
||||
)
|
||||
|
||||
# 写 materials_content（占位：file_path 写入，extracted_text 待 parse-materials）
|
||||
# 写 materials_content（sandbox card 文本 + parse-materials 解析后文件路径）
|
||||
from db.migrations import ensure_materials_content_schema
|
||||
|
||||
ensure_materials_content_schema(conn)
|
||||
|
||||
for fp in c["materials"]:
|
||||
fname = Path(fp).name
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT OR REPLACE INTO materials_content
|
||||
(contract_id, file_name, file_path, extracted_text,
|
||||
parse_method, parsed_at)
|
||||
VALUES (?, ?, ?, NULL, 'pending', ?)
|
||||
""",
|
||||
(
|
||||
c["contract_id"],
|
||||
fname,
|
||||
fp,
|
||||
datetime.now().isoformat(timespec="seconds"),
|
||||
),
|
||||
)
|
||||
for m in c["materials"]:
|
||||
if isinstance(m, dict) and m.get("kind") == "sandbox_card":
|
||||
# sandbox mock 卡片文本，直接写入 extracted_text
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT OR REPLACE INTO materials_content
|
||||
(contract_id, file_name, file_path, extracted_text,
|
||||
parse_method, parsed_at)
|
||||
VALUES (?, ?, ?, ?, 'sandbox_card', ?)
|
||||
""",
|
||||
(
|
||||
c["contract_id"],
|
||||
m["file_name"],
|
||||
f"sandbox://hr/contracts/{c['contract_id']}",
|
||||
m["text"],
|
||||
datetime.now().isoformat(timespec="seconds"),
|
||||
),
|
||||
)
|
||||
# 同步解析 → 抽 year + 等次 → 写入 performance_records
|
||||
perf_n = repo.parse_sandbox_card_performance(
|
||||
conn, c["contract_id"], m["text"],
|
||||
)
|
||||
if perf_n > 0:
|
||||
logger.info(
|
||||
"platform_collector: %s 解析卡片 → performance %d 条",
|
||||
c["contract_id"], perf_n,
|
||||
)
|
||||
else:
|
||||
# 本地文件路径（parse-materials 解析后写入）
|
||||
fp = m if isinstance(m, str) else m.get("file_path")
|
||||
fname = Path(fp).name if fp else "unknown"
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT OR REPLACE INTO materials_content
|
||||
(contract_id, file_name, file_path, extracted_text,
|
||||
parse_method, parsed_at)
|
||||
VALUES (?, ?, ?, NULL, 'pending', ?)
|
||||
""",
|
||||
(
|
||||
c["contract_id"],
|
||||
fname,
|
||||
fp,
|
||||
datetime.now().isoformat(timespec="seconds"),
|
||||
),
|
||||
)
|
||||
return len(contracts)
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -450,11 +537,14 @@ def _error_result(reason: str, fetched: int) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _gen_contract_id(teacher: str, end_date: str) -> str:
|
||||
"""从 (姓名, 到期日) 派 contract_id，确保幂等。"""
|
||||
safe_teacher = re.sub(r"[^\w]", "", teacher)[:8] or "T"
|
||||
def _gen_contract_id(emp_id: str, end_date: str) -> str:
|
||||
"""从 (工号, 到期日) 派 contract_id，确保幂等。
|
||||
|
||||
sandbox v1.1.1 起改用「工号」作主键（之前用姓名）。
|
||||
"""
|
||||
safe_emp = re.sub(r"[^\w]", "", emp_id)[:10] or "T"
|
||||
safe_end = end_date.replace("-", "")[:8]
|
||||
return f"RPA-{safe_teacher}-{safe_end}"
|
||||
return f"RPA-{safe_emp}-{safe_end}"
|
||||
|
||||
|
||||
def _normalize_date(s: str) -> str:
|
||||
|
||||
Reference in New Issue
Block a user