diff --git a/scripts/db/repository.py b/scripts/db/repository.py index 3accc7b..3e2bc9e 100644 --- a/scripts/db/repository.py +++ b/scripts/db/repository.py @@ -59,6 +59,82 @@ def list_performance_by_contract( ).fetchall() +# 等次 → 分数映射（v1.1.1 sandbox card 解析） +_GRADE_TO_SCORE = { + "优秀": 90.0, + "称职": 75.0, + "基本称职": 60.0, + "不称职": 30.0, + "不合格": 30.0, +} + + +def upsert_performance_record( + conn: sqlite3.Connection, + contract_id: str, + year: int, + grade: str, +) -> bool: + """upsert 一条业绩记录（用 grade 映射分数）。 + + sandbox card 文本不含真分数时使用。返回是否实际写入。 + """ + score = _GRADE_TO_SCORE.get(grade) + if score is None: + return False + record_id = f"P-{contract_id}-{year}" + conn.execute( + """ + INSERT INTO performance_records + (record_id, contract_id, year, performance_score, + moral_teaching_score, research_score, teaching_score) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(record_id) DO UPDATE SET + performance_score = excluded.performance_score + """, + (record_id, contract_id, year, score, score, score, score), + ) + return True + + +def parse_sandbox_card_performance( + conn: sqlite3.Connection, + contract_id: str, + card_text: str, +) -> int: + """从 sandbox 卡片文本抽 (year, 等次) → upsert 到 performance_records。 + + 文本片段形如： + year result + 2024 称职 + 2025 优秀 + 2024年：等次「称职」。 + 返回写入条数。 + """ + import re + if not card_text: + return 0 + + written = 0 + # 模式 1：行格式 "2024\t称职" + for m in re.finditer(r"^(\d{4})\s*[ \t]+(优秀|称职|基本称职|不称职|不合格)\s*$", + card_text, re.MULTILINE): + year = int(m.group(1)) + grade = m.group(2) + if upsert_performance_record(conn, contract_id, year, grade): + written += 1 + + # 模式 2：叙述式 "2024年：等次「称职」" + for m in re.finditer(r"(\d{4})\s*年[：:]\s*等次[「\"']\s*(优秀|称职|基本称职|不称职|不合格)\s*[」\"']", + card_text): + year = int(m.group(1)) + grade = m.group(2) + if upsert_performance_record(conn, contract_id, year, grade): + written += 1 + + return written + + # ---------- alerts ---------- def insert_alert( diff --git a/scripts/service/platform_collector.py b/scripts/service/platform_collector.py index 79c1989..826436a 100644 --- a/scripts/service/platform_collector.py +++ b/scripts/service/platform_collector.py @@ -243,6 +243,17 @@ def collect_all(limit: int = 0, headless: bool = True) -> dict: logger.info("platform_collector: 当前 URL=%s", page.url) + # 5.5. 进入页面后刷新（SPA 进入新路由时 mock 数据可能未加载，需刷新触发） + try: + logger.info("platform_collector: 进入 hr/contracts 后刷新页面") + page.reload(wait_until="networkidle", timeout=20_000) + except Exception as e: + logger.warning("刷新 networkidle 失败: %s；尝试 domcontentloaded", e) + try: + page.reload(wait_until="domcontentloaded", timeout=20_000) + except Exception as e2: + logger.warning("刷新 domcontentloaded 也失败: %s", e2) + # 7. dump 当前 hr/contracts 页面（用于调试） try: html = page.content() @@ -284,6 +295,23 @@ def collect_all(limit: int = 0, headless: bool = True) -> dict: logger.warning("dump 失败: %s", e) return _error_result("table_not_found_after_40s", 0) + # 表格已找到，等 tbody 完全填充 + dump 一份完整 DOM（用于诊断按钮 selector） + try: + page.wait_for_function( + "document.querySelectorAll('table tbody tr').length > 0", + timeout=10_000, + ) + page.wait_for_timeout(1500) + html = page.content() + full_dump = download_dir.parent / "hr_contracts_full_dump.html" + full_dump.write_text(html, encoding="utf-8") + logger.info( + "platform_collector: 完整 hr/contracts dump → %s (%d 字节)", + full_dump, len(html), + ) + except Exception as e: + logger.warning("等完整渲染超时: %s", e) + rows = page.eval_on_selector_all( "table tbody tr", "rs => rs.map(r => Array.from(r.querySelectorAll('td')).map(td => td.innerText))", @@ -302,21 +330,33 @@ def collect_all(limit: int = 0, headless: bool = True) -> dict: except Exception: mat_btn_count = 0 + # 抓「材料包」card 区段的 DOM（用于后续解析每行 click 后的卡片文本） + # sandbox 行为：click 按钮后会在表格下方追加「续聘准备任务书」+「单位清单」卡片 + for i, r in enumerate(rows): if limit and i >= limit: break - if len(r) < 6: - logger.warning("行 %d 列数 < 6，跳过: %s", i, r) + if len(r) < 7: + logger.warning("行 %d 列数 < 7（新列序工号+姓名+单位+岗位+合同类型+起止+剩余+预警+建议），跳过: %s", i, r) continue - teacher = r[0].strip() - unit = r[1].strip() - contract_end = r[2].strip() - days_raw = r[3].strip() - alert_label = r[4].strip() - suggestion_hint = r[5].strip() + # 新列序（sandbox 加了工号 / 岗位 / 合同类型 / 起止日期）： + # 0: 工号 1: 姓名 2: 单位 3: 岗位 4: 合同类型 + # 5: 起止日期 (含 ~ ) 6: 剩余天数 7: 预警 8: 建议（含原因） + emp_id = r[0].strip() + teacher = r[1].strip() + unit = r[2].strip() + position = r[3].strip() + contract_type = r[4].strip() + period = r[5].strip() + # 起止日期形如 "2024-03-01 ~ 2026-04-15" → 取 ~ 后 + contract_end = period.split("~")[-1].strip() if "~" in period else period + days_raw = r[6].strip() + alert_label = r[7].strip() + suggestion_hint = r[8].strip() if len(r) > 8 else "" - contract_id = _gen_contract_id(teacher, contract_end) + # 用工号生成 contract_id（稳定，不会随姓名变化） + contract_id = _gen_contract_id(emp_id, contract_end) contract = { "contract_id": contract_id, "teacher_name": teacher, @@ -328,32 +368,50 @@ def collect_all(limit: int = 0, headless: bool = True) -> dict: "materials": [], } - # 只有在按钮存在时才尝试下载（sandbox mock 当前未部署此按钮） + # 点「续聘材料包」按钮 → 等卡片展开 → 抓卡片文本（含 checklist） if mat_btn_count > 0: try: btn = page.locator("table tbody tr").nth(i).locator( "button:has-text('续聘材料包')" ) - with page.expect_download(timeout=10_000) as dl_info: - btn.click() - dl = dl_info.value - fname = dl.suggested_filename or "materials.zip" - dest = download_dir / f"{contract_id}_{fname}" - dl.save_as(str(dest)) - contract["materials"].append(str(dest)) - downloaded += 1 + btn.click() + # 等卡片出现（页面下方追加 h2「续聘准备任务书」/「单位清单」） + page.wait_for_timeout(1500) + # 抓表格下方所有 card 文本 + cards_text = page.locator( + "main .grid > div, main > div > div > div" + ).all_inner_texts() + materials_text = "\n\n".join( + t.strip() for t in cards_text if t.strip() + ) + if materials_text: + # 存到 materials_content（sandbox mock 卡片文本作为材料包内容） + contract["materials"].append({ + "kind": "sandbox_card", + "file_name": f"{contract_id}_materials.txt", + "text": materials_text, + }) + downloaded += 1 + logger.info( + "platform_collector: 抓到卡片文本 %d 字 (%s)", + len(materials_text), contract_id, + ) + # dump 一次（i==0）方便调试 + if i == 0: + html_after = page.content() + click_dump = download_dir.parent / "after_click_dump.html" + click_dump.write_text(html_after, encoding="utf-8") + logger.info( + "platform_collector: 首行 click 后 dump → %s (%d 字节)", + click_dump, len(html_after), + ) except Exception as e: - logger.warning("材料下载失败 %s: %s", contract_id, e) + logger.warning("抓卡片失败 %s: %s", contract_id, e) else: - logger.info( - "platform_collector: sandbox 未部署「续聘材料包」按钮，" - "跳过材料下载（仅采集表格列）", - ) - # 仅警告一次 if i == 0: - logger.warning( - "platform_collector: 提示：材料包缺失时，" - "parse-materials CLI 将标记为「无材料」" + logger.info( + "platform_collector: sandbox 未部署「续聘材料包」按钮，" + "跳过材料抓取（仅采集表格列）", ) contracts.append(contract) @@ -413,27 +471,56 @@ def _persist_contracts(contracts: list[dict]) -> int: ), ) - # 写 materials_content（占位：file_path 写入，extracted_text 待 parse-materials） + # 写 materials_content（sandbox card 文本 + parse-materials 解析后文件路径） from db.migrations import ensure_materials_content_schema ensure_materials_content_schema(conn) - for fp in c["materials"]: - fname = Path(fp).name - conn.execute( - """ - INSERT OR REPLACE INTO materials_content - (contract_id, file_name, file_path, extracted_text, - parse_method, parsed_at) - VALUES (?, ?, ?, NULL, 'pending', ?) - """, - ( - c["contract_id"], - fname, - fp, - datetime.now().isoformat(timespec="seconds"), - ), - ) + for m in c["materials"]: + if isinstance(m, dict) and m.get("kind") == "sandbox_card": + # sandbox mock 卡片文本，直接写入 extracted_text + conn.execute( + """ + INSERT OR REPLACE INTO materials_content + (contract_id, file_name, file_path, extracted_text, + parse_method, parsed_at) + VALUES (?, ?, ?, ?, 'sandbox_card', ?) + """, + ( + c["contract_id"], + m["file_name"], + f"sandbox://hr/contracts/{c['contract_id']}", + m["text"], + datetime.now().isoformat(timespec="seconds"), + ), + ) + # 同步解析 → 抽 year + 等次 → 写入 performance_records + perf_n = repo.parse_sandbox_card_performance( + conn, c["contract_id"], m["text"], + ) + if perf_n > 0: + logger.info( + "platform_collector: %s 解析卡片 → performance %d 条", + c["contract_id"], perf_n, + ) + else: + # 本地文件路径（parse-materials 解析后写入） + fp = m if isinstance(m, str) else m.get("file_path") + fname = Path(fp).name if fp else "unknown" + conn.execute( + """ + INSERT OR REPLACE INTO materials_content + (contract_id, file_name, file_path, extracted_text, + parse_method, parsed_at) + VALUES (?, ?, ?, NULL, 'pending', ?) + """, + ( + c["contract_id"], + fname, + fp, + datetime.now().isoformat(timespec="seconds"), + ), + ) return len(contracts) finally: conn.close() @@ -450,11 +537,14 @@ def _error_result(reason: str, fetched: int) -> dict: } -def _gen_contract_id(teacher: str, end_date: str) -> str: - """从 (姓名, 到期日) 派 contract_id，确保幂等。""" - safe_teacher = re.sub(r"[^\w]", "", teacher)[:8] or "T" +def _gen_contract_id(emp_id: str, end_date: str) -> str: + """从 (工号, 到期日) 派 contract_id，确保幂等。 + + sandbox v1.1.1 起改用「工号」作主键（之前用姓名）。 + """ + safe_emp = re.sub(r"[^\w]", "", emp_id)[:10] or "T" safe_end = end_date.replace("-", "")[:8] - return f"RPA-{safe_teacher}-{safe_end}" + return f"RPA-{safe_emp}-{safe_end}" def _normalize_date(s: str) -> str: