v1.1.2: sandbox 卡卡片文本采集 + 解析为 performance 记录
sandbox 行为新发现:
1. 「续聘材料包」按钮不是下载文件,而是触发页面下方显示「续聘准备任务书」
+「单位清单」卡片(含 checklist + 近三年考核表)
2. 进入 hr/contracts 页面后需要刷新(page.reload)才能看到表格行
3. 表格列序扩展为 10 列:工号 / 姓名 / 单位 / 岗位 / 合同类型 / 起止日期 / 剩余天数 / 预警 / 建议 / 材料包按钮
4. contract_id 改用工号(稳定,不随姓名变化)
改动:
- platform_collector.collect_all:
* 进入 hr/contracts 后 page.reload 刷新触发 SPA mock 数据加载
* 新列序映射:r[0]=工号 r[1]=姓名 r[2]=单位 r[3]=岗位 r[4]=合同类型
r[5]=起止日期 r[6]=剩余天数 r[7]=预警 r[8]=建议
* 合同 end_date 从「2024-03-01 ~ 2026-04-15」截取 ~ 后的部分
* 点「续聘材料包」按钮 → 抓下方卡片 inner_text(含 checklist + 考核等次)
* 把卡片文本存入 materials_content(parse_method='sandbox_card')
- _persist_contracts 增加 sandbox_card 路径分支:写 materials_content 后同步调
repo.parse_sandbox_card_performance → 抽 year + 等次 → 写 performance_records
- repository.parse_sandbox_card_performance:双模式正则匹配
* 行格式 "2024\t称职"
* 叙述式 "2024年:等次「称职」"
- repository.upsert_performance_record + 等次 → 分数映射
(优秀=90 称职=75 基本称职=60 不称职=30)
实测(sandbox 3 合同):
- RPA-T20190033-20260415 钱实验 → 解析 4 条 performance
- RPA-T20200055-20260630 孙助理 → 解析 6 条
- RPA-T20180021-20260831 王明远 → 解析 6 条
- generate-renewal-task 全部输出 observe + decision_source=rules
Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
@@ -59,6 +59,82 @@ def list_performance_by_contract(
|
|||||||
).fetchall()
|
).fetchall()
|
||||||
|
|
||||||
|
|
||||||
|
# 等次 → 分数映射(v1.1.1 sandbox card 解析)
|
||||||
|
_GRADE_TO_SCORE = {
|
||||||
|
"优秀": 90.0,
|
||||||
|
"称职": 75.0,
|
||||||
|
"基本称职": 60.0,
|
||||||
|
"不称职": 30.0,
|
||||||
|
"不合格": 30.0,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def upsert_performance_record(
|
||||||
|
conn: sqlite3.Connection,
|
||||||
|
contract_id: str,
|
||||||
|
year: int,
|
||||||
|
grade: str,
|
||||||
|
) -> bool:
|
||||||
|
"""upsert 一条业绩记录(用 grade 映射分数)。
|
||||||
|
|
||||||
|
sandbox card 文本不含真分数时使用。返回是否实际写入。
|
||||||
|
"""
|
||||||
|
score = _GRADE_TO_SCORE.get(grade)
|
||||||
|
if score is None:
|
||||||
|
return False
|
||||||
|
record_id = f"P-{contract_id}-{year}"
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO performance_records
|
||||||
|
(record_id, contract_id, year, performance_score,
|
||||||
|
moral_teaching_score, research_score, teaching_score)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(record_id) DO UPDATE SET
|
||||||
|
performance_score = excluded.performance_score
|
||||||
|
""",
|
||||||
|
(record_id, contract_id, year, score, score, score, score),
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def parse_sandbox_card_performance(
|
||||||
|
conn: sqlite3.Connection,
|
||||||
|
contract_id: str,
|
||||||
|
card_text: str,
|
||||||
|
) -> int:
|
||||||
|
"""从 sandbox 卡片文本抽 (year, 等次) → upsert 到 performance_records。
|
||||||
|
|
||||||
|
文本片段形如:
|
||||||
|
year result
|
||||||
|
2024 称职
|
||||||
|
2025 优秀
|
||||||
|
2024年:等次「称职」。
|
||||||
|
返回写入条数。
|
||||||
|
"""
|
||||||
|
import re
|
||||||
|
if not card_text:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
written = 0
|
||||||
|
# 模式 1:行格式 "2024\t称职"
|
||||||
|
for m in re.finditer(r"^(\d{4})\s*[ \t]+(优秀|称职|基本称职|不称职|不合格)\s*$",
|
||||||
|
card_text, re.MULTILINE):
|
||||||
|
year = int(m.group(1))
|
||||||
|
grade = m.group(2)
|
||||||
|
if upsert_performance_record(conn, contract_id, year, grade):
|
||||||
|
written += 1
|
||||||
|
|
||||||
|
# 模式 2:叙述式 "2024年:等次「称职」"
|
||||||
|
for m in re.finditer(r"(\d{4})\s*年[::]\s*等次[「\"']\s*(优秀|称职|基本称职|不称职|不合格)\s*[」\"']",
|
||||||
|
card_text):
|
||||||
|
year = int(m.group(1))
|
||||||
|
grade = m.group(2)
|
||||||
|
if upsert_performance_record(conn, contract_id, year, grade):
|
||||||
|
written += 1
|
||||||
|
|
||||||
|
return written
|
||||||
|
|
||||||
|
|
||||||
# ---------- alerts ----------
|
# ---------- alerts ----------
|
||||||
|
|
||||||
def insert_alert(
|
def insert_alert(
|
||||||
|
|||||||
@@ -243,6 +243,17 @@ def collect_all(limit: int = 0, headless: bool = True) -> dict:
|
|||||||
|
|
||||||
logger.info("platform_collector: 当前 URL=%s", page.url)
|
logger.info("platform_collector: 当前 URL=%s", page.url)
|
||||||
|
|
||||||
|
# 5.5. 进入页面后刷新(SPA 进入新路由时 mock 数据可能未加载,需刷新触发)
|
||||||
|
try:
|
||||||
|
logger.info("platform_collector: 进入 hr/contracts 后刷新页面")
|
||||||
|
page.reload(wait_until="networkidle", timeout=20_000)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("刷新 networkidle 失败: %s;尝试 domcontentloaded", e)
|
||||||
|
try:
|
||||||
|
page.reload(wait_until="domcontentloaded", timeout=20_000)
|
||||||
|
except Exception as e2:
|
||||||
|
logger.warning("刷新 domcontentloaded 也失败: %s", e2)
|
||||||
|
|
||||||
# 7. dump 当前 hr/contracts 页面(用于调试)
|
# 7. dump 当前 hr/contracts 页面(用于调试)
|
||||||
try:
|
try:
|
||||||
html = page.content()
|
html = page.content()
|
||||||
@@ -284,6 +295,23 @@ def collect_all(limit: int = 0, headless: bool = True) -> dict:
|
|||||||
logger.warning("dump 失败: %s", e)
|
logger.warning("dump 失败: %s", e)
|
||||||
return _error_result("table_not_found_after_40s", 0)
|
return _error_result("table_not_found_after_40s", 0)
|
||||||
|
|
||||||
|
# 表格已找到,等 tbody 完全填充 + dump 一份完整 DOM(用于诊断按钮 selector)
|
||||||
|
try:
|
||||||
|
page.wait_for_function(
|
||||||
|
"document.querySelectorAll('table tbody tr').length > 0",
|
||||||
|
timeout=10_000,
|
||||||
|
)
|
||||||
|
page.wait_for_timeout(1500)
|
||||||
|
html = page.content()
|
||||||
|
full_dump = download_dir.parent / "hr_contracts_full_dump.html"
|
||||||
|
full_dump.write_text(html, encoding="utf-8")
|
||||||
|
logger.info(
|
||||||
|
"platform_collector: 完整 hr/contracts dump → %s (%d 字节)",
|
||||||
|
full_dump, len(html),
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("等完整渲染超时: %s", e)
|
||||||
|
|
||||||
rows = page.eval_on_selector_all(
|
rows = page.eval_on_selector_all(
|
||||||
"table tbody tr",
|
"table tbody tr",
|
||||||
"rs => rs.map(r => Array.from(r.querySelectorAll('td')).map(td => td.innerText))",
|
"rs => rs.map(r => Array.from(r.querySelectorAll('td')).map(td => td.innerText))",
|
||||||
@@ -302,21 +330,33 @@ def collect_all(limit: int = 0, headless: bool = True) -> dict:
|
|||||||
except Exception:
|
except Exception:
|
||||||
mat_btn_count = 0
|
mat_btn_count = 0
|
||||||
|
|
||||||
|
# 抓「材料包」card 区段的 DOM(用于后续解析每行 click 后的卡片文本)
|
||||||
|
# sandbox 行为:click 按钮后会在表格下方追加「续聘准备任务书」+「单位清单」卡片
|
||||||
|
|
||||||
for i, r in enumerate(rows):
|
for i, r in enumerate(rows):
|
||||||
if limit and i >= limit:
|
if limit and i >= limit:
|
||||||
break
|
break
|
||||||
if len(r) < 6:
|
if len(r) < 7:
|
||||||
logger.warning("行 %d 列数 < 6,跳过: %s", i, r)
|
logger.warning("行 %d 列数 < 7(新列序工号+姓名+单位+岗位+合同类型+起止+剩余+预警+建议),跳过: %s", i, r)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
teacher = r[0].strip()
|
# 新列序(sandbox 加了工号 / 岗位 / 合同类型 / 起止日期):
|
||||||
unit = r[1].strip()
|
# 0: 工号 1: 姓名 2: 单位 3: 岗位 4: 合同类型
|
||||||
contract_end = r[2].strip()
|
# 5: 起止日期 (含 ~ ) 6: 剩余天数 7: 预警 8: 建议(含原因)
|
||||||
days_raw = r[3].strip()
|
emp_id = r[0].strip()
|
||||||
alert_label = r[4].strip()
|
teacher = r[1].strip()
|
||||||
suggestion_hint = r[5].strip()
|
unit = r[2].strip()
|
||||||
|
position = r[3].strip()
|
||||||
|
contract_type = r[4].strip()
|
||||||
|
period = r[5].strip()
|
||||||
|
# 起止日期形如 "2024-03-01 ~ 2026-04-15" → 取 ~ 后
|
||||||
|
contract_end = period.split("~")[-1].strip() if "~" in period else period
|
||||||
|
days_raw = r[6].strip()
|
||||||
|
alert_label = r[7].strip()
|
||||||
|
suggestion_hint = r[8].strip() if len(r) > 8 else ""
|
||||||
|
|
||||||
contract_id = _gen_contract_id(teacher, contract_end)
|
# 用工号生成 contract_id(稳定,不会随姓名变化)
|
||||||
|
contract_id = _gen_contract_id(emp_id, contract_end)
|
||||||
contract = {
|
contract = {
|
||||||
"contract_id": contract_id,
|
"contract_id": contract_id,
|
||||||
"teacher_name": teacher,
|
"teacher_name": teacher,
|
||||||
@@ -328,32 +368,50 @@ def collect_all(limit: int = 0, headless: bool = True) -> dict:
|
|||||||
"materials": [],
|
"materials": [],
|
||||||
}
|
}
|
||||||
|
|
||||||
# 只有在按钮存在时才尝试下载(sandbox mock 当前未部署此按钮)
|
# 点「续聘材料包」按钮 → 等卡片展开 → 抓卡片文本(含 checklist)
|
||||||
if mat_btn_count > 0:
|
if mat_btn_count > 0:
|
||||||
try:
|
try:
|
||||||
btn = page.locator("table tbody tr").nth(i).locator(
|
btn = page.locator("table tbody tr").nth(i).locator(
|
||||||
"button:has-text('续聘材料包')"
|
"button:has-text('续聘材料包')"
|
||||||
)
|
)
|
||||||
with page.expect_download(timeout=10_000) as dl_info:
|
btn.click()
|
||||||
btn.click()
|
# 等卡片出现(页面下方追加 h2「续聘准备任务书」/「单位清单」)
|
||||||
dl = dl_info.value
|
page.wait_for_timeout(1500)
|
||||||
fname = dl.suggested_filename or "materials.zip"
|
# 抓表格下方所有 card 文本
|
||||||
dest = download_dir / f"{contract_id}_{fname}"
|
cards_text = page.locator(
|
||||||
dl.save_as(str(dest))
|
"main .grid > div, main > div > div > div"
|
||||||
contract["materials"].append(str(dest))
|
).all_inner_texts()
|
||||||
downloaded += 1
|
materials_text = "\n\n".join(
|
||||||
|
t.strip() for t in cards_text if t.strip()
|
||||||
|
)
|
||||||
|
if materials_text:
|
||||||
|
# 存到 materials_content(sandbox mock 卡片文本作为材料包内容)
|
||||||
|
contract["materials"].append({
|
||||||
|
"kind": "sandbox_card",
|
||||||
|
"file_name": f"{contract_id}_materials.txt",
|
||||||
|
"text": materials_text,
|
||||||
|
})
|
||||||
|
downloaded += 1
|
||||||
|
logger.info(
|
||||||
|
"platform_collector: 抓到卡片文本 %d 字 (%s)",
|
||||||
|
len(materials_text), contract_id,
|
||||||
|
)
|
||||||
|
# dump 一次(i==0)方便调试
|
||||||
|
if i == 0:
|
||||||
|
html_after = page.content()
|
||||||
|
click_dump = download_dir.parent / "after_click_dump.html"
|
||||||
|
click_dump.write_text(html_after, encoding="utf-8")
|
||||||
|
logger.info(
|
||||||
|
"platform_collector: 首行 click 后 dump → %s (%d 字节)",
|
||||||
|
click_dump, len(html_after),
|
||||||
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("材料下载失败 %s: %s", contract_id, e)
|
logger.warning("抓卡片失败 %s: %s", contract_id, e)
|
||||||
else:
|
else:
|
||||||
logger.info(
|
|
||||||
"platform_collector: sandbox 未部署「续聘材料包」按钮,"
|
|
||||||
"跳过材料下载(仅采集表格列)",
|
|
||||||
)
|
|
||||||
# 仅警告一次
|
|
||||||
if i == 0:
|
if i == 0:
|
||||||
logger.warning(
|
logger.info(
|
||||||
"platform_collector: 提示:材料包缺失时,"
|
"platform_collector: sandbox 未部署「续聘材料包」按钮,"
|
||||||
"parse-materials CLI 将标记为「无材料」"
|
"跳过材料抓取(仅采集表格列)",
|
||||||
)
|
)
|
||||||
|
|
||||||
contracts.append(contract)
|
contracts.append(contract)
|
||||||
@@ -413,27 +471,56 @@ def _persist_contracts(contracts: list[dict]) -> int:
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
# 写 materials_content(占位:file_path 写入,extracted_text 待 parse-materials)
|
# 写 materials_content(sandbox card 文本 + parse-materials 解析后文件路径)
|
||||||
from db.migrations import ensure_materials_content_schema
|
from db.migrations import ensure_materials_content_schema
|
||||||
|
|
||||||
ensure_materials_content_schema(conn)
|
ensure_materials_content_schema(conn)
|
||||||
|
|
||||||
for fp in c["materials"]:
|
for m in c["materials"]:
|
||||||
fname = Path(fp).name
|
if isinstance(m, dict) and m.get("kind") == "sandbox_card":
|
||||||
conn.execute(
|
# sandbox mock 卡片文本,直接写入 extracted_text
|
||||||
"""
|
conn.execute(
|
||||||
INSERT OR REPLACE INTO materials_content
|
"""
|
||||||
(contract_id, file_name, file_path, extracted_text,
|
INSERT OR REPLACE INTO materials_content
|
||||||
parse_method, parsed_at)
|
(contract_id, file_name, file_path, extracted_text,
|
||||||
VALUES (?, ?, ?, NULL, 'pending', ?)
|
parse_method, parsed_at)
|
||||||
""",
|
VALUES (?, ?, ?, ?, 'sandbox_card', ?)
|
||||||
(
|
""",
|
||||||
c["contract_id"],
|
(
|
||||||
fname,
|
c["contract_id"],
|
||||||
fp,
|
m["file_name"],
|
||||||
datetime.now().isoformat(timespec="seconds"),
|
f"sandbox://hr/contracts/{c['contract_id']}",
|
||||||
),
|
m["text"],
|
||||||
)
|
datetime.now().isoformat(timespec="seconds"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
# 同步解析 → 抽 year + 等次 → 写入 performance_records
|
||||||
|
perf_n = repo.parse_sandbox_card_performance(
|
||||||
|
conn, c["contract_id"], m["text"],
|
||||||
|
)
|
||||||
|
if perf_n > 0:
|
||||||
|
logger.info(
|
||||||
|
"platform_collector: %s 解析卡片 → performance %d 条",
|
||||||
|
c["contract_id"], perf_n,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
# 本地文件路径(parse-materials 解析后写入)
|
||||||
|
fp = m if isinstance(m, str) else m.get("file_path")
|
||||||
|
fname = Path(fp).name if fp else "unknown"
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT OR REPLACE INTO materials_content
|
||||||
|
(contract_id, file_name, file_path, extracted_text,
|
||||||
|
parse_method, parsed_at)
|
||||||
|
VALUES (?, ?, ?, NULL, 'pending', ?)
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
c["contract_id"],
|
||||||
|
fname,
|
||||||
|
fp,
|
||||||
|
datetime.now().isoformat(timespec="seconds"),
|
||||||
|
),
|
||||||
|
)
|
||||||
return len(contracts)
|
return len(contracts)
|
||||||
finally:
|
finally:
|
||||||
conn.close()
|
conn.close()
|
||||||
@@ -450,11 +537,14 @@ def _error_result(reason: str, fetched: int) -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _gen_contract_id(teacher: str, end_date: str) -> str:
|
def _gen_contract_id(emp_id: str, end_date: str) -> str:
|
||||||
"""从 (姓名, 到期日) 派 contract_id,确保幂等。"""
|
"""从 (工号, 到期日) 派 contract_id,确保幂等。
|
||||||
safe_teacher = re.sub(r"[^\w]", "", teacher)[:8] or "T"
|
|
||||||
|
sandbox v1.1.1 起改用「工号」作主键(之前用姓名)。
|
||||||
|
"""
|
||||||
|
safe_emp = re.sub(r"[^\w]", "", emp_id)[:10] or "T"
|
||||||
safe_end = end_date.replace("-", "")[:8]
|
safe_end = end_date.replace("-", "")[:8]
|
||||||
return f"RPA-{safe_teacher}-{safe_end}"
|
return f"RPA-{safe_emp}-{safe_end}"
|
||||||
|
|
||||||
|
|
||||||
def _normalize_date(s: str) -> str:
|
def _normalize_date(s: str) -> str:
|
||||||
|
|||||||
Reference in New Issue
Block a user