112 lines
4.0 KiB
Python
112 lines
4.0 KiB
Python
"""主动扫描 data/input/{contracts,materials}/：CSV → contracts 表，PDF/Word → materials_content 表。
|
||
|
||
对 sandbox 用户尤其重要：HR API 拿不到时，把材料直接丢进 input/materials/ 即可被自动识别。
|
||
|
||
约定：
|
||
- input/contracts/*.csv
|
||
列名与 contracts 表一致（contract_id / teacher_name / unit_name / contract_end / ...）
|
||
也兼容中文别名（合同ID / 教职工姓名 / 单位 / 合同到期日 / 工号 / 岗位 / 合同类型）
|
||
- input/materials/{contract_id}__{filename}.{pdf|docx|doc}
|
||
文件名 __ 之前是 contract_id；无 __ 时回退 parent dir 名
|
||
拷贝到 data/materials/ + insert materials_content(parse_method='pending')
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import csv
|
||
import shutil
|
||
import sqlite3
|
||
from pathlib import Path
|
||
|
||
from db import repository as repo
|
||
from db.migrations import ensure_materials_content_schema
|
||
from util.logging_setup import get_skill_logger
|
||
from util.runtime_paths import (
|
||
get_data_dir,
|
||
get_input_contracts_dir,
|
||
get_input_materials_dir,
|
||
)
|
||
|
||
logger = get_skill_logger()
|
||
|
||
MATERIAL_EXTS = {".pdf", ".docx", ".doc"}
|
||
|
||
|
||
def _derive_contract_id_from_filename(mat_path: Path) -> str:
|
||
"""从文件名推导 contract_id。约定：{contract_id}__{filename}.{ext}。"""
|
||
parts = mat_path.stem.split("__", 1)
|
||
if len(parts) == 2 and parts[0]:
|
||
return parts[0]
|
||
# 兜底：parent dir 名
|
||
return mat_path.parent.name or "UNKNOWN"
|
||
|
||
|
||
def import_input_dir(conn: sqlite3.Connection) -> dict:
|
||
"""扫描 input 目录 → 入库。返回统计 dict。
|
||
|
||
Returns:
|
||
{
|
||
"contracts_added": int,
|
||
"materials_added": int,
|
||
"files_scanned": int,
|
||
"errors": [str, ...],
|
||
}
|
||
"""
|
||
ensure_materials_content_schema(conn)
|
||
|
||
contracts_dir = get_input_contracts_dir()
|
||
materials_dir = get_input_materials_dir()
|
||
|
||
contracts_added = 0
|
||
materials_added = 0
|
||
files_scanned = 0
|
||
errors: list[str] = []
|
||
|
||
# 1. CSV → contracts（contract_id 重复 → upsert 覆盖）
|
||
for csv_path in sorted(contracts_dir.glob("*.csv")):
|
||
files_scanned += 1
|
||
try:
|
||
with open(csv_path, "r", encoding="utf-8", newline="") as f:
|
||
reader = csv.DictReader(f)
|
||
for row in reader:
|
||
if not row.get("contract_id") and not row.get("合同ID"):
|
||
errors.append(f"{csv_path.name}: row 缺 contract_id 跳过")
|
||
continue
|
||
repo.upsert_contract_from_csv_row(conn, row)
|
||
contracts_added += 1
|
||
except Exception as e:
|
||
errors.append(f"{csv_path.name}: {e}")
|
||
logger.warning("importer: CSV 解析失败 %s: %s", csv_path, e)
|
||
|
||
# 2. PDF/Word → materials_content（parse_method='pending'），文件拷贝到 data/materials/
|
||
materials_target_dir = get_data_dir() / "materials"
|
||
materials_target_dir.mkdir(parents=True, exist_ok=True)
|
||
for mat_path in sorted(materials_dir.iterdir()):
|
||
if not mat_path.is_file() or mat_path.suffix.lower() not in MATERIAL_EXTS:
|
||
continue
|
||
files_scanned += 1
|
||
try:
|
||
contract_id = _derive_contract_id_from_filename(mat_path)
|
||
target = materials_target_dir / mat_path.name
|
||
if not target.exists():
|
||
shutil.copy2(mat_path, target)
|
||
repo.insert_material(
|
||
conn, contract_id, mat_path.name, str(target),
|
||
None, "pending",
|
||
)
|
||
materials_added += 1
|
||
logger.info(
|
||
"importer: 入库 %s → contract_id=%s", mat_path.name, contract_id,
|
||
)
|
||
except Exception as e:
|
||
errors.append(f"{mat_path.name}: {e}")
|
||
logger.warning("importer: 材料导入失败 %s: %s", mat_path, e)
|
||
|
||
conn.commit()
|
||
return {
|
||
"contracts_added": contracts_added,
|
||
"materials_added": materials_added,
|
||
"files_scanned": files_scanned,
|
||
"errors": errors,
|
||
}
|