chore: auto release commit (2026-09-15 00:32:44)

scripts/service/config_viewer.py

@@ -0,0 +1,77 @@
+"""读取并脱敏 .env 配置供 UI 查看（v1.1.18）。
+
+仅显示 .env 文件中的非空 key；命中 (TOKEN|SECRET|KEY|PASSWORD) 的字段自动脱敏为 ***。
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from util.runtime_paths import get_env_path
+
+
+# 匹配任一含 TOKEN / SECRET / KEY / PASSWORD 的 key 名（不区分大小写）
+_SECRET_PATTERN = re.compile(r"(TOKEN|SECRET|KEY|PASSWORD)", re.IGNORECASE)
+
+
+def _parse_env_file(path: Path) -> dict[str, str]:
+    """手动解析 .env（避免依赖 python-dotenv）。
+
+    - 跳过空行 / 注释（# 开头）
+    - 支持 KEY=VALUE 格式
+    - 自动去除包裹的 "..." 或 '...'
+    """
+    if not path.exists():
+        return {}
+    result: dict[str, str] = {}
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            if "=" not in line:
+                continue
+            k, _, v = line.partition("=")
+            k = k.strip()
+            v = v.strip()
+            # 去掉行尾注释（不处理引号内的 #，简单实现）
+            if " #" in v:
+                v = v.split(" #", 1)[0].strip()
+            if v.startswith('"') and v.endswith('"') and len(v) >= 2:
+                v = v[1:-1]
+            elif v.startswith("'") and v.endswith("'") and len(v) >= 2:
+                v = v[1:-1]
+            result[k] = v
+    return result
+
+
+def _mask_secret(value: str) -> str:
+    if not value:
+        return ""
+    if len(value) <= 4:
+        return "***"
+    return value[:2] + "***" + value[-2:]
+
+
+def read_env_config() -> dict:
+    """读取 .env 中的非空 key，敏感 key 脱敏为 ***。
+
+    Returns:
+        {
+            "env_path": str,
+            "exists": bool,
+            "config": {key: value_or_masked_value, ...}
+        }
+    """
+    path = get_env_path()
+    parsed = _parse_env_file(path)
+    out: dict[str, str] = {}
+    for k, v in sorted(parsed.items()):
+        if not v:
+            continue
+        if _SECRET_PATTERN.search(k):
+            out[k] = _mask_secret(v)
+        else:
+            out[k] = v
+    return {"env_path": str(path), "exists": path.exists(), "config": out}

scripts/service/importer.py

@@ -0,0 +1,111 @@
+"""主动扫描 data/input/{contracts,materials}/：CSV → contracts 表，PDF/Word → materials_content 表。
+
+对 sandbox 用户尤其重要：HR API 拿不到时，把材料直接丢进 input/materials/ 即可被自动识别。
+
+约定：
+  - input/contracts/*.csv
+      列名与 contracts 表一致（contract_id / teacher_name / unit_name / contract_end / ...）
+      也兼容中文别名（合同ID / 教职工姓名 / 单位 / 合同到期日 / 工号 / 岗位 / 合同类型）
+  - input/materials/{contract_id}__{filename}.{pdf|docx|doc}
+      文件名 __ 之前是 contract_id；无 __ 时回退 parent dir 名
+      拷贝到 data/materials/ + insert materials_content(parse_method='pending')
+"""
+
+from __future__ import annotations
+
+import csv
+import shutil
+import sqlite3
+from pathlib import Path
+
+from db import repository as repo
+from db.migrations import ensure_materials_content_schema
+from util.logging_setup import get_skill_logger
+from util.runtime_paths import (
+    get_data_dir,
+    get_input_contracts_dir,
+    get_input_materials_dir,
+)
+
+logger = get_skill_logger()
+
+MATERIAL_EXTS = {".pdf", ".docx", ".doc"}
+
+
+def _derive_contract_id_from_filename(mat_path: Path) -> str:
+    """从文件名推导 contract_id。约定：{contract_id}__{filename}.{ext}。"""
+    parts = mat_path.stem.split("__", 1)
+    if len(parts) == 2 and parts[0]:
+        return parts[0]
+    # 兜底：parent dir 名
+    return mat_path.parent.name or "UNKNOWN"
+
+
+def import_input_dir(conn: sqlite3.Connection) -> dict:
+    """扫描 input 目录 → 入库。返回统计 dict。
+
+    Returns:
+        {
+            "contracts_added": int,
+            "materials_added": int,
+            "files_scanned": int,
+            "errors": [str, ...],
+        }
+    """
+    ensure_materials_content_schema(conn)
+
+    contracts_dir = get_input_contracts_dir()
+    materials_dir = get_input_materials_dir()
+
+    contracts_added = 0
+    materials_added = 0
+    files_scanned = 0
+    errors: list[str] = []
+
+    # 1. CSV → contracts（contract_id 重复 → upsert 覆盖）
+    for csv_path in sorted(contracts_dir.glob("*.csv")):
+        files_scanned += 1
+        try:
+            with open(csv_path, "r", encoding="utf-8", newline="") as f:
+                reader = csv.DictReader(f)
+                for row in reader:
+                    if not row.get("contract_id") and not row.get("合同ID"):
+                        errors.append(f"{csv_path.name}: row 缺 contract_id 跳过")
+                        continue
+                    repo.upsert_contract_from_csv_row(conn, row)
+                    contracts_added += 1
+        except Exception as e:
+            errors.append(f"{csv_path.name}: {e}")
+            logger.warning("importer: CSV 解析失败 %s: %s", csv_path, e)
+
+    # 2. PDF/Word → materials_content（parse_method='pending'），文件拷贝到 data/materials/
+    materials_target_dir = get_data_dir() / "materials"
+    materials_target_dir.mkdir(parents=True, exist_ok=True)
+    for mat_path in sorted(materials_dir.iterdir()):
+        if not mat_path.is_file() or mat_path.suffix.lower() not in MATERIAL_EXTS:
+            continue
+        files_scanned += 1
+        try:
+            contract_id = _derive_contract_id_from_filename(mat_path)
+            target = materials_target_dir / mat_path.name
+            if not target.exists():
+                shutil.copy2(mat_path, target)
+            repo.insert_material(
+                conn, contract_id, mat_path.name, str(target),
+                None, "pending",
+            )
+            materials_added += 1
+            logger.info(
+                "importer: 入库 %s → contract_id=%s", mat_path.name, contract_id,
+            )
+        except Exception as e:
+            errors.append(f"{mat_path.name}: {e}")
+            logger.warning("importer: 材料导入失败 %s: %s", mat_path, e)
+
+    conn.commit()
+    return {
+        "contracts_added": contracts_added,
+        "materials_added": materials_added,
+        "files_scanned": files_scanned,
+        "errors": errors,
+    }

scripts/service/logs_viewer.py

@@ -0,0 +1,86 @@
+"""读取日志文件末尾 N 行（v1.1.18）。
+
+按 level 过滤：默认 INFO+。适用于 RotatingFileHandler 落地的日志文件。
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from util.runtime_paths import get_logs_dir
+
+
+DEFAULT_TAIL = 200
+_LEVEL_ORDER = {
+    "DEBUG": 10, "INFO": 20, "WARNING": 30, "ERROR": 40, "CRITICAL": 50,
+}
+
+
+def _resolve_log_path() -> Path:
+    """解析日志文件路径。
+
+    优先级：
+      1. LOG_FILE env（绝对路径直接用，相对路径走 get_logs_dir()）
+      2. 默认 get_logs_dir() / LOG_FILE_DEFAULT_NAME
+    """
+    raw = os.environ.get("LOG_FILE", "")
+    if raw:
+        p = Path(raw)
+        if not p.is_absolute():
+            p = get_logs_dir() / raw
+        return p
+    from util.constants import LOG_FILE_DEFAULT_NAME
+    return get_logs_dir() / LOG_FILE_DEFAULT_NAME
+
+
+def tail_log_file(tail: int = DEFAULT_TAIL, min_level: str = "INFO") -> dict:
+    """读日志文件末尾 tail 行，过滤 level >= min_level。
+
+    Returns:
+        {
+            "log_file": str,
+            "exists": bool,
+            "total_lines": int,
+            "returned_lines": int,
+            "lines": [str, ...],
+            "min_level": str,
+        }
+    """
+    log_path = _resolve_log_path()
+    if not log_path.exists():
+        return {
+            "log_file": str(log_path),
+            "exists": False,
+            "total_lines": 0,
+            "returned_lines": 0,
+            "lines": [],
+            "min_level": min_level,
+        }
+
+    text = log_path.read_text(encoding="utf-8", errors="replace")
+    all_lines = text.splitlines()
+    last_n = all_lines[-tail:] if tail > 0 else all_lines
+    threshold = _LEVEL_ORDER.get(min_level.upper(), 20)
+
+    filtered: list[str] = []
+    for line in last_n:
+        # 格式: "2026-09-15 12:34:56 [alert-contract-expiring-hr] INFO message"
+        # 取第 4 个 token (INFO/DEBUG/...)
+        parts = line.split()
+        if len(parts) >= 4 and parts[1].count(":") >= 2 and parts[3] in _LEVEL_ORDER:
+            line_level = parts[3]
+            if _LEVEL_ORDER.get(line_level, 0) >= threshold:
+                filtered.append(line)
+        else:
+            # 非标准行 / 解析不到 level：保留
+            filtered.append(line)
+
+    return {
+        "log_file": str(log_path),
+        "exists": True,
+        "total_lines": len(all_lines),
+        "returned_lines": len(filtered),
+        "lines": filtered,
+        "min_level": min_level,
+    }

scripts/service/platform_collector.py

@@ -865,11 +865,265 @@ def _persist_contracts(contracts: list[dict]) -> int:
                                 datetime.now().isoformat(timespec="seconds"),
                             ),
                         )
+                # v1.1.18：每个合同入队 download_tasks（即使已抓到卡片也入队，
+                # process_queue 失败时可用作重试锚点；enqueue_download_task 幂等）
+                repo.enqueue_download_task(conn, c["contract_id"], "platform_button")
         return len(contracts)
     finally:
         conn.close()
 
 
+def _download_materials_for_row(
+    page, row_idx: int, contract_id: str, mat_btn_count: int,
+) -> list[dict]:
+    """点续聘材料包按钮抓卡片文本（与 v1.1.17 collect_all 内联逻辑等价）。
+
+    返回 materials 列表（kind=sandbox_card，含 file_name + text）。
+    """
+    if mat_btn_count == 0:
+        return []
+    materials: list[dict] = []
+    try:
+        btn = page.locator("table tbody tr").nth(row_idx).locator(
+            "button:has-text('续聘材料包')"
+        )
+        btn.click()
+        page.wait_for_timeout(CARD_APPEAR_WAIT_MS)
+        cards_text = page.locator(
+            "main .grid > div, main > div > div > div"
+        ).all_inner_texts()
+        materials_text = "\n\n".join(t.strip() for t in cards_text if t.strip())
+        if materials_text:
+            materials.append({
+                "kind": "sandbox_card",
+                "file_name": f"{contract_id}_materials.txt",
+                "text": materials_text,
+            })
+            logger.info(
+                "platform_collector: 抓到卡片文本 %d 字 (%s)",
+                len(materials_text), contract_id,
+            )
+    except Exception as e:
+        logger.warning("抓卡片失败 %s: %s", contract_id, e)
+    return materials
+
+
+def _find_row_for_contract(page, contract_id: str) -> int | None:
+    """在 hr/contracts 表格中找 contract_id 所在行号。
+
+    支持 7 列（用 teacher_name 派生 contract_id）与 10 列（用 emp_id）。
+    不匹配返回 None（调用方 mark_task_failed）。
+    """
+    rows = page.eval_on_selector_all(
+        "table tbody tr",
+        "rs => rs.map(r => Array.from(r.querySelectorAll('td')).map(td => td.innerText))",
+    )
+    for i, r in enumerate(rows):
+        ncols = len(r)
+        if ncols < 6:
+            continue
+        first_col = r[0].strip()
+        is_emp_id = bool(re.match(r"^[A-Za-z]?\d+$", first_col))
+        if is_emp_id and ncols >= 9:
+            emp_id, contract_end = first_col, r[5].split("~")[-1].strip()
+        elif ncols >= 6:
+            emp_id, contract_end = first_col, r[2].strip()
+        else:
+            continue
+        candidate = _gen_contract_id(emp_id, contract_end)
+        if candidate == contract_id:
+            return i
+    return None
+
+
+def process_queue(
+    limit: int = 5,
+    headless: bool = True,
+    human_wait_timeout: int = HUMAN_WAIT_TIMEOUT,
+    ids_filter: str = "",
+) -> dict:
+    """从 download_tasks 队列取 pending 任务，对每个合同重新走材料抓取。
+
+    与 collect_all 重复登录流程，但每行只点「续聘材料包」按钮抓卡片，不重新抓合同表。
+
+    参数：
+      limit: 本次最多处理任务数（0 = 不限）
+      ids_filter: 逗号分隔 task_id 列表（只处理这些 pending）
+      headless: 无头模式（需要 profile cookie 已保存）
+      human_wait_timeout: 手工登录 fallback 等待秒数
+
+    返回 {processed, succeeded, failed, details}。
+    """
+    try:
+        from playwright.sync_api import sync_playwright
+    except ImportError:
+        logger.error("playwright 未安装，跳过队列处理")
+        return {"processed": 0, "succeeded": 0, "failed": 0, "details": [],
+                "error": "playwright_not_installed"}
+
+    from db.connection import get_connection
+
+    # 1. 领取一批任务
+    ids_list = [s.strip() for s in ids_filter.split(",") if s.strip()] if ids_filter else None
+    conn = get_connection()
+    try:
+        tasks = repo.claim_pending_tasks(
+            conn,
+            limit=limit if limit > 0 else 999,
+            ids_filter=ids_list,
+        )
+    finally:
+        conn.close()
+
+    if not tasks:
+        return {"processed": 0, "succeeded": 0, "failed": 0, "details": []}
+
+    profile_dir = _safe_getenv_dir("SMARTCAMPUS_PROFILE_DIR", "playwright-profile")
+    download_dir = _safe_getenv_dir("MATERIALS_DOWNLOAD_DIR", "materials")
+    profile_dir.mkdir(parents=True, exist_ok=True)
+    download_dir.mkdir(parents=True, exist_ok=True)
+
+    details: list[dict] = []
+    succeeded = 0
+    failed = 0
+
+    with sync_playwright() as p:
+        ctx = p.chromium.launch_persistent_context(
+            user_data_dir=str(profile_dir),
+            headless=headless,
+            accept_downloads=True,
+            viewport={"width": 1440, "height": 900},
+            channel="msedge" if not headless else None,
+        ) if not headless else p.chromium.launch_persistent_context(
+            user_data_dir=str(profile_dir),
+            headless=True,
+            accept_downloads=True,
+            viewport={"width": 1440, "height": 900},
+        )
+        try:
+            page = ctx.new_page()
+            page.set_default_timeout(DEFAULT_PAGE_TIMEOUT_MS)
+
+            # 登录：复用 v1.1.17 collect_all 的探针 + 自动 fill + 手工 fallback 流程
+            logger.info("platform_collector.process_queue: 启动 profile=%s headless=%s 待处理 %d",
+                        profile_dir, headless, len(tasks))
+            _goto(page, SPA_HOME_URL)
+            page.wait_for_timeout(SPA_HYDRATION_MS)
+            if _is_logged_in(page):
+                logger.info("platform_collector.process_queue: ✅ 已登录态")
+            else:
+                _goto(page, SPA_LOGIN_URL)
+                page.wait_for_timeout(SPA_HYDRATION_MS)
+                if not _fill_spa_login(page):
+                    if not headless:
+                        _wait_for_manual_login(page, wait_sec=human_wait_timeout)
+                    else:
+                        raise LoginTimeoutError("process_queue headless 自动 fill 失败")
+                else:
+                    page.wait_for_timeout(POST_LOGIN_SETTLE_MS)
+
+            # 强制 goto SPA_HOME + 侧边栏 hr/contracts
+            for _ in range(SPA_HOME_RETRY_MAX):
+                if _is_logged_in(page):
+                    break
+                _goto(page, SPA_HOME_URL)
+                page.wait_for_timeout(SPA_HOME_RETRY_DELAY_MS)
+                if _fill_spa_login(page):
+                    page.wait_for_timeout(POST_LOGIN_SETTLE_MS)
+            if not _is_logged_in(page):
+                raise LoginTimeoutError("process_queue 仍未登录")
+
+            # 等表格
+            page.wait_for_timeout(TABLE_INITIAL_WAIT_MS)
+            found_table = False
+            for _attempt in range(TABLE_POLL_MAX_ATTEMPTS):
+                page.wait_for_timeout(TABLE_POLL_INTERVAL_MS)
+                try:
+                    page.wait_for_selector(
+                        "table tbody tr", timeout=TABLE_TR_TIMEOUT_MS,
+                    )
+                    found_table = True
+                    break
+                except Exception:
+                    pass
+            if not found_table:
+                raise TableNotFoundError("process_queue 表格未找到")
+
+            mat_btn_count = page.locator(
+                "table tbody tr button:has-text('续聘材料包')"
+            ).count()
+
+            # 2. 逐任务处理
+            for t in tasks:
+                contract_id = t["contract_id"]
+                task_id = t["task_id"]
+                conn = get_connection()
+                try:
+                    row_idx = _find_row_for_contract(page, contract_id)
+                    if row_idx is None:
+                        repo.mark_task_failed(conn, task_id, "contract_not_found_in_table")
+                        failed += 1
+                        details.append({"task_id": task_id, "contract_id": contract_id,
+                                        "error": "contract_not_found_in_table"})
+                        conn.commit()
+                        continue
+                    materials = _download_materials_for_row(
+                        page, row_idx, contract_id, mat_btn_count,
+                    )
+                    from db.migrations import ensure_materials_content_schema
+                    ensure_materials_content_schema(conn)
+                    for m in materials:
+                        if isinstance(m, dict) and m.get("kind") == "sandbox_card":
+                            conn.execute(
+                                """
+                                INSERT OR REPLACE INTO materials_content
+                                    (contract_id, file_name, file_path, extracted_text,
+                                     parse_method, parsed_at)
+                                VALUES (?, ?, ?, ?, 'sandbox_card', ?)
+                                """,
+                                (
+                                    contract_id, m["file_name"],
+                                    f"sandbox://hr/contracts/{contract_id}",
+                                    m["text"],
+                                    datetime.now().isoformat(timespec="seconds"),
+                                ),
+                            )
+                            repo.parse_sandbox_card_performance(conn, contract_id, m["text"])
+                    result_path = (
+                        f"sandbox://hr/contracts/{contract_id}"
+                        if materials else "no_materials_captured"
+                    )
+                    repo.mark_task_success(conn, task_id, result_path)
+                    succeeded += 1
+                    details.append({
+                        "task_id": task_id,
+                        "contract_id": contract_id,
+                        "ok": True,
+                        "materials_count": len(materials),
+                    })
+                    conn.commit()
+                except Exception as e:
+                    repo.mark_task_failed(conn, task_id, str(e))
+                    failed += 1
+                    details.append({
+                        "task_id": task_id, "contract_id": contract_id,
+                        "error": str(e),
+                    })
+                    conn.commit()
+                    logger.warning("process_queue: task=%s 失败: %s", task_id, e)
+                finally:
+                    conn.close()
+        finally:
+            ctx.close()
+
+    return {
+        "processed": len(tasks),
+        "succeeded": succeeded,
+        "failed": failed,
+        "details": details,
+    }
+
+
 def _error_result(reason: str, fetched: int) -> dict:
     return {
         "platform_url": PLATFORM_URL,