"""主动扫描 data/input/{contracts,materials}/:CSV → contracts 表,PDF/Word → materials_content 表。 对 sandbox 用户尤其重要:HR API 拿不到时,把材料直接丢进 input/materials/ 即可被自动识别。 约定: - input/contracts/*.csv 列名与 contracts 表一致(contract_id / teacher_name / unit_name / contract_end / ...) 也兼容中文别名(合同ID / 教职工姓名 / 单位 / 合同到期日 / 工号 / 岗位 / 合同类型) - input/materials/{contract_id}__{filename}.{pdf|docx|doc} 文件名 __ 之前是 contract_id;无 __ 时回退 parent dir 名 拷贝到 data/materials/ + insert materials_content(parse_method='pending') """ from __future__ import annotations import csv import shutil import sqlite3 from pathlib import Path from db import repository as repo from db.migrations import ensure_materials_content_schema from util.logging_setup import get_skill_logger from util.runtime_paths import ( get_data_dir, get_input_contracts_dir, get_input_materials_dir, ) logger = get_skill_logger() MATERIAL_EXTS = {".pdf", ".docx", ".doc"} def _derive_contract_id_from_filename(mat_path: Path) -> str: """从文件名推导 contract_id。约定:{contract_id}__{filename}.{ext}。""" parts = mat_path.stem.split("__", 1) if len(parts) == 2 and parts[0]: return parts[0] # 兜底:parent dir 名 return mat_path.parent.name or "UNKNOWN" def import_input_dir(conn: sqlite3.Connection) -> dict: """扫描 input 目录 → 入库。返回统计 dict。 Returns: { "contracts_added": int, "materials_added": int, "files_scanned": int, "errors": [str, ...], } """ ensure_materials_content_schema(conn) contracts_dir = get_input_contracts_dir() materials_dir = get_input_materials_dir() contracts_added = 0 materials_added = 0 files_scanned = 0 errors: list[str] = [] # 1. CSV → contracts(contract_id 重复 → upsert 覆盖) for csv_path in sorted(contracts_dir.glob("*.csv")): files_scanned += 1 try: with open(csv_path, "r", encoding="utf-8", newline="") as f: reader = csv.DictReader(f) for row in reader: if not row.get("contract_id") and not row.get("合同ID"): errors.append(f"{csv_path.name}: row 缺 contract_id 跳过") continue repo.upsert_contract_from_csv_row(conn, row) contracts_added += 1 except Exception as e: errors.append(f"{csv_path.name}: {e}") logger.warning("importer: CSV 解析失败 %s: %s", csv_path, e) # 2. PDF/Word → materials_content(parse_method='pending'),文件拷贝到 data/materials/ materials_target_dir = get_data_dir() / "materials" materials_target_dir.mkdir(parents=True, exist_ok=True) for mat_path in sorted(materials_dir.iterdir()): if not mat_path.is_file() or mat_path.suffix.lower() not in MATERIAL_EXTS: continue files_scanned += 1 try: contract_id = _derive_contract_id_from_filename(mat_path) target = materials_target_dir / mat_path.name if not target.exists(): shutil.copy2(mat_path, target) repo.insert_material( conn, contract_id, mat_path.name, str(target), None, "pending", ) materials_added += 1 logger.info( "importer: 入库 %s → contract_id=%s", mat_path.name, contract_id, ) except Exception as e: errors.append(f"{mat_path.name}: {e}") logger.warning("importer: 材料导入失败 %s: %s", mat_path, e) conn.commit() return { "contracts_added": contracts_added, "materials_added": materials_added, "files_scanned": files_scanned, "errors": errors, }