440 lines
18 KiB
Python
440 lines
18 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""默认测试套件：课程教学大纲合规性体检。
|
||
|
||
设计原则：
|
||
- 仅用标准库（不依赖 pytest / jinja2 / python-docx），CI 与本地均能直接
|
||
`python tests/run_tests.py` 跑
|
||
- v1.0.1 仍为骨架版，业务实现细节待后续补全；本套件聚焦：
|
||
- util.constants 版本对齐（POLICY-RELEASE-001）
|
||
- util.score_util 纯函数（clamp / weighted_total / classify_audit_status /
|
||
parse_score_threshold / parse_ideology_keywords / count_keyword_hits）
|
||
- scripts/main.py CLI 占位命令（health / version / config-path / init-db /
|
||
audit-syllabus / batch-audit-syllabus / render-html / sync-norm-library）
|
||
- 退出码：0 = 全部通过，1 = 有失败
|
||
- -v 开关：详细输出每个用例的状态
|
||
|
||
release.ps1 在打 tag 前会自动跑 `python tests/run_tests.py -v`，任何一个
|
||
test_* 返回 (False, msg) 即 abort 发布。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import re
|
||
import sys
|
||
from pathlib import Path
|
||
from typing import Callable
|
||
|
||
# 让 scripts/ 下的模块可导入
|
||
_REPO_ROOT = Path(__file__).resolve().parent.parent
|
||
_SCRIPTS_DIR = _REPO_ROOT / "scripts"
|
||
sys.path.insert(0, str(_SCRIPTS_DIR))
|
||
|
||
|
||
# ---------- 测试基础设施 ----------
|
||
|
||
_RESULTS: list[tuple[str, bool, str]] = []
|
||
_VERBOSE = False
|
||
|
||
|
||
def _run(name: str, fn: Callable[[], tuple[bool, str]]) -> None:
|
||
"""运行单个 test_*，记录结果。"""
|
||
import traceback
|
||
try:
|
||
ok, msg = fn()
|
||
except Exception as e: # 测试本身抛异常也算失败
|
||
tb = traceback.format_exc(limit=3)
|
||
ok, msg = False, f"exception: {e}\n{tb}"
|
||
_RESULTS.append((name, ok, msg))
|
||
if _VERBOSE or not ok:
|
||
marker = "PASS" if ok else "FAIL"
|
||
print(f" [{marker}] {name}: {msg}", flush=True)
|
||
|
||
|
||
# ---------- util.constants 版本对齐（POLICY-RELEASE-001） ----------
|
||
|
||
def test_skill_md_version_alignment() -> tuple[bool, str]:
|
||
"""POLICY-RELEASE-001：SKILL.md version == SKILL_VERSION == CHANGELOG 顶部版本"""
|
||
from util.constants import SKILL_VERSION
|
||
|
||
skill_md = (_REPO_ROOT / "SKILL.md").read_text(encoding="utf-8")
|
||
m = re.search(r'(?m)^version:\s*["\']?(\d+\.\d+\.\d+)', skill_md)
|
||
if not m:
|
||
return False, "SKILL.md 缺顶层 version: X.Y.Z"
|
||
md_ver = m.group(1)
|
||
if md_ver != SKILL_VERSION:
|
||
return False, f"SKILL.md={md_ver} != SKILL_VERSION={SKILL_VERSION}"
|
||
|
||
# CHANGELOG 顶部应有 ## X.Y.Z
|
||
cl = (_REPO_ROOT / "CHANGELOG.md").read_text(encoding="utf-8")
|
||
cl_m = re.search(r'(?m)^##\s+(\d+\.\d+\.\d+)', cl)
|
||
if not cl_m:
|
||
return False, "CHANGELOG.md 缺顶层 ## X.Y.Z"
|
||
cl_ver = cl_m.group(1)
|
||
if cl_ver != SKILL_VERSION:
|
||
return False, f"CHANGELOG 顶部={cl_ver} != SKILL_VERSION={SKILL_VERSION}"
|
||
|
||
return True, f"SKILL.md={md_ver} == CHANGELOG={cl_ver} == SKILL_VERSION={SKILL_VERSION}"
|
||
|
||
|
||
def test_skill_md_slug_alignment() -> tuple[bool, str]:
|
||
"""SKILL.md metadata.openclaw.slug == SKILL_SLUG"""
|
||
from util.constants import SKILL_SLUG
|
||
|
||
skill_md = (_REPO_ROOT / "SKILL.md").read_text(encoding="utf-8")
|
||
m = re.search(r'(?m)^\s*slug:\s*["\']?([a-z0-9_-]+)', skill_md)
|
||
if not m:
|
||
return False, "SKILL.md 缺 metadata.openclaw.slug"
|
||
md_slug = m.group(1)
|
||
if md_slug != SKILL_SLUG:
|
||
return False, f"SKILL.md slug={md_slug} != SKILL_SLUG={SKILL_SLUG}"
|
||
return True, f"slug={md_slug}"
|
||
|
||
|
||
def test_constants_threshold_defaults() -> tuple[bool, str]:
|
||
"""util.constants 中的阈值常量符合 SPEC §6 输入要求（默认 85 / 60 / 60 秒）"""
|
||
from util.constants import (
|
||
DEFAULT_AUDIT_TIMEOUT_SEC,
|
||
DEFAULT_MATRIX_MATCH_THRESHOLD,
|
||
DEFAULT_SCORE_MODIFY_THRESHOLD,
|
||
DEFAULT_SCORE_PASS_THRESHOLD,
|
||
DEFAULT_WEIGHT_FORMAT,
|
||
DEFAULT_WEIGHT_IDEOLOGY,
|
||
DEFAULT_WEIGHT_MATRIX,
|
||
)
|
||
|
||
if DEFAULT_SCORE_PASS_THRESHOLD != 85:
|
||
return False, f"DEFAULT_SCORE_PASS_THRESHOLD={DEFAULT_SCORE_PASS_THRESHOLD}, want 85"
|
||
if DEFAULT_SCORE_MODIFY_THRESHOLD != 60:
|
||
return False, f"DEFAULT_SCORE_MODIFY_THRESHOLD={DEFAULT_SCORE_MODIFY_THRESHOLD}, want 60"
|
||
if DEFAULT_AUDIT_TIMEOUT_SEC != 60:
|
||
return False, f"DEFAULT_AUDIT_TIMEOUT_SEC={DEFAULT_AUDIT_TIMEOUT_SEC}, want 60"
|
||
if DEFAULT_MATRIX_MATCH_THRESHOLD != 0.7:
|
||
return False, f"DEFAULT_MATRIX_MATCH_THRESHOLD={DEFAULT_MATRIX_MATCH_THRESHOLD}, want 0.7"
|
||
|
||
w_sum = DEFAULT_WEIGHT_FORMAT + DEFAULT_WEIGHT_MATRIX + DEFAULT_WEIGHT_IDEOLOGY
|
||
if abs(w_sum - 1.0) > 1e-9:
|
||
return False, f"三项权重和={w_sum}, want 1.0"
|
||
return True, "PASS=85 / MODIFY=60 / TIMEOUT=60 / MATRIX=0.7 / 权重和=1.0"
|
||
|
||
|
||
# ---------- util.score_util 纯函数 ----------
|
||
|
||
def test_clamp_score() -> tuple[bool, str]:
|
||
from util.score_util import clamp_score
|
||
cases = [
|
||
(50, 50.0),
|
||
(-10, 0.0),
|
||
(150, 100.0),
|
||
(0, 0.0),
|
||
(100, 100.0),
|
||
(99.5, 99.5),
|
||
("abc", 0.0), # 非法字符串 → 0
|
||
(None, 0.0), # None → 0
|
||
("85.5", 85.5), # 数字字符串
|
||
]
|
||
for inp, want in cases:
|
||
got = clamp_score(inp)
|
||
if got != want:
|
||
return False, f"clamp_score({inp!r}) = {got}, want {want}"
|
||
return True, "9 场景（正常 / 越界 / 非法 / None / 数字字符串）"
|
||
|
||
|
||
def test_weighted_total_default() -> tuple[bool, str]:
|
||
"""默认权重 (0.3 / 0.4 / 0.3)：90 / 80 / 70 → 0.3*90+0.4*80+0.3*70 = 80"""
|
||
from util.score_util import weighted_total
|
||
got = weighted_total(90, 80, 70)
|
||
want = 80.0
|
||
if got != want:
|
||
return False, f"weighted_total(90,80,70) = {got}, want {want}"
|
||
return True, "默认权重 (0.3/0.4/0.3) 加权求和 = 80.0"
|
||
|
||
|
||
def test_weighted_total_overrides() -> tuple[bool, str]:
|
||
"""自定义权重 (0.5 / 0.3 / 0.2)：100 / 50 / 50 → 50+15+10 = 75"""
|
||
from util.score_util import weighted_total
|
||
got = weighted_total(100, 50, 50, weights={"format": 0.5, "matrix": 0.3, "ideology": 0.2})
|
||
if got != 75.0:
|
||
return False, f"weighted_total 自定义权重 = {got}, want 75.0"
|
||
return True, "自定义权重求和 = 75.0"
|
||
|
||
|
||
def test_weighted_total_clamps_overshoot() -> tuple[bool, str]:
|
||
"""超界输入会被裁剪：999 / 999 / 999 → 100*1.0 = 100"""
|
||
from util.score_util import weighted_total
|
||
got = weighted_total(999, 999, 999)
|
||
if got != 100.0:
|
||
return False, f"weighted_total 超界 = {got}, want 100.0"
|
||
return True, "三项均超 100 → 裁剪到 100"
|
||
|
||
|
||
def test_classify_audit_status_pass() -> tuple[bool, str]:
|
||
"""score >= pass_thr → pass"""
|
||
from util.score_util import classify_audit_status
|
||
# 边界 85 == 85 → pass
|
||
if classify_audit_status(85, 85, 60) != "pass":
|
||
return False, "85/85/60 应为 pass"
|
||
if classify_audit_status(100, 85, 60) != "pass":
|
||
return False, "100 应为 pass"
|
||
return True, "≥ pass_thr → pass（边界 + 远超）"
|
||
|
||
|
||
def test_classify_audit_status_modify() -> tuple[bool, str]:
|
||
"""modify_thr ≤ score < pass_thr → modify"""
|
||
from util.score_util import classify_audit_status
|
||
if classify_audit_status(60, 85, 60) != "modify":
|
||
return False, "60（modify 边界）应为 modify"
|
||
if classify_audit_status(80, 85, 60) != "modify":
|
||
return False, "80 应为 modify"
|
||
if classify_audit_status(84.99, 85, 60) != "modify":
|
||
return False, "84.99 应为 modify"
|
||
return True, "modify_thr ≤ score < pass_thr → modify（边界 + 中段）"
|
||
|
||
|
||
def test_classify_audit_status_rewrite() -> tuple[bool, str]:
|
||
"""score < modify_thr → rewrite"""
|
||
from util.score_util import classify_audit_status
|
||
if classify_audit_status(0, 85, 60) != "rewrite":
|
||
return False, "0 应为 rewrite"
|
||
if classify_audit_status(59.99, 85, 60) != "rewrite":
|
||
return False, "59.99 应为 rewrite"
|
||
if classify_audit_status(-10, 85, 60) != "rewrite":
|
||
return False, "负分应为 rewrite"
|
||
return True, "< modify_thr → rewrite（边界 / 负数 / 0）"
|
||
|
||
|
||
def test_classify_audit_status_swapped_thresholds() -> tuple[bool, str]:
|
||
"""pass_thr < modify_thr 时防御性交换（保证函数依然正确）"""
|
||
from util.score_util import classify_audit_status
|
||
# 传入颠倒阈值：pass=60, modify=85 → 应被视作 pass=85, modify=60
|
||
# 70 ≥ 60 但 < 85 → modify
|
||
if classify_audit_status(70, 60, 85) != "modify":
|
||
return False, "颠倒阈值 70 应为 modify（防御性交换后）"
|
||
if classify_audit_status(90, 60, 85) != "pass":
|
||
return False, "颠倒阈值 90 应为 pass"
|
||
return True, "pass_thr < modify_thr 防御性交换后正确"
|
||
|
||
|
||
def test_parse_score_threshold() -> tuple[bool, str]:
|
||
from util.score_util import parse_score_threshold
|
||
cases = [
|
||
("85", 85.0),
|
||
("60", 60.0),
|
||
(" 75 ", 75.0), # 带空白
|
||
("", 85.0), # 空 → default
|
||
(None, 85.0), # None → default
|
||
("abc", 85.0), # 非法 → default
|
||
("150", 85.0), # 超界 → default
|
||
("-5", 85.0), # 负数 → default
|
||
]
|
||
for inp, want in cases:
|
||
got = parse_score_threshold(inp, default=85.0)
|
||
if got != want:
|
||
return False, f"parse_score_threshold({inp!r}) = {got}, want {want}"
|
||
return True, "8 场景（合法 / 空白 / 空 / None / 非法 / 超界 / 负数）"
|
||
|
||
|
||
def test_parse_ideology_keywords_default() -> tuple[bool, str]:
|
||
"""空 / None → 默认关键词列表（与 .env.example 一致：家国情怀 / 科学伦理 / 工匠精神 / 社会责任）"""
|
||
from util.score_util import parse_ideology_keywords
|
||
default = ["家国情怀", "科学伦理", "工匠精神", "社会责任"]
|
||
got_empty = parse_ideology_keywords("", default)
|
||
got_none = parse_ideology_keywords(None, default)
|
||
if got_empty != default:
|
||
return False, f"空字符串应回退 default，实测 {got_empty}"
|
||
if got_none != default:
|
||
return False, f"None 应回退 default，实测 {got_none}"
|
||
return True, "空 / None → 默认关键词"
|
||
|
||
|
||
def test_parse_ideology_keywords_json() -> tuple[bool, str]:
|
||
"""JSON 数组字符串正常解析"""
|
||
from util.score_util import parse_ideology_keywords
|
||
default = ["家国情怀"]
|
||
s = '["家国情怀", "科学伦理", "工匠精神"]'
|
||
got = parse_ideology_keywords(s, default)
|
||
if got != ["家国情怀", "科学伦理", "工匠精神"]:
|
||
return False, f"JSON 解析结果 {got}"
|
||
return True, 'JSON 数组 → ["家国情怀","科学伦理","工匠精神"]'
|
||
|
||
|
||
def test_parse_ideology_keywords_invalid() -> tuple[bool, str]:
|
||
"""非法 JSON / 非 list / 全空元素 → 回退 default"""
|
||
from util.score_util import parse_ideology_keywords
|
||
default = ["家国情怀", "科学伦理"]
|
||
cases = [
|
||
"garbage", # 非法 JSON
|
||
'{"a":1}', # 不是 list
|
||
"[1, 2, 3]", # 元素不是 str
|
||
"[null, '', ' ']", # 全空 str
|
||
'[" ", "\t"]', # 仅空白
|
||
]
|
||
for s in cases:
|
||
got = parse_ideology_keywords(s, default)
|
||
if got != default:
|
||
return False, f"parse_ideology_keywords({s!r}) = {got}, want default"
|
||
return True, "5 场景（非法 JSON / 非 list / 非 str / 全空）→ default"
|
||
|
||
|
||
def test_count_keyword_hits_basic() -> tuple[bool, str]:
|
||
"""统计思政关键词在文本中的命中次数"""
|
||
from util.score_util import count_keyword_hits
|
||
text = "本课程培养学生家国情怀，强调工匠精神与社会责任"
|
||
keywords = ["家国情怀", "科学伦理", "工匠精神", "社会责任"]
|
||
got = count_keyword_hits(text, keywords)
|
||
want = {"家国情怀": 1, "工匠精神": 1, "社会责任": 1}
|
||
if got != want:
|
||
return False, f"count_keyword_hits = {got}, want {want}"
|
||
return True, "3 命中（家国情怀/工匠精神/社会责任），科学伦理 0"
|
||
|
||
|
||
def test_count_keyword_hits_repeats() -> tuple[bool, str]:
|
||
"""同一关键词多次出现应累计"""
|
||
from util.score_util import count_keyword_hits
|
||
text = "家国情怀 家国情怀 家国情怀。工匠精神。"
|
||
keywords = ["家国情怀", "工匠精神"]
|
||
got = count_keyword_hits(text, keywords)
|
||
if got.get("家国情怀") != 3:
|
||
return False, f"家国情怀命中={got.get('家国情怀')}, want 3"
|
||
if got.get("工匠精神") != 1:
|
||
return False, f"工匠精神命中={got.get('工匠精神')}, want 1"
|
||
return True, "多次出现累计（3 / 1）"
|
||
|
||
|
||
def test_count_keyword_hits_edge_cases() -> tuple[bool, str]:
|
||
"""边界：空文本 / 空关键词列表 / 非 str 输入 → 返回空 dict，不报错"""
|
||
from util.score_util import count_keyword_hits
|
||
if count_keyword_hits("", ["家国情怀"]) != {}:
|
||
return False, "空文本应返回 {}"
|
||
if count_keyword_hits("任何文本", []) != {}:
|
||
return False, "空关键词应返回 {}"
|
||
if count_keyword_hits(None, ["家国情怀"]) != {}:
|
||
return False, "非 str 文本应返回 {}"
|
||
if count_keyword_hits("家国情怀", [None, "", "家国情怀"]) != {"家国情怀": 1}:
|
||
return False, "关键词中含 None / 空串应被跳过"
|
||
return True, "空文本 / 空关键词 / 非 str / 关键词含 None — 全部安全"
|
||
|
||
|
||
# ---------- scripts.main CLI 占位命令 ----------
|
||
|
||
def test_cli_command_set() -> tuple[bool, str]:
|
||
"""main.py COMMANDS 应包含 SPEC §0.1 的全部入口"""
|
||
import importlib
|
||
|
||
main_mod = importlib.import_module("main")
|
||
COMMANDS = getattr(main_mod, "COMMANDS", {})
|
||
expected = {
|
||
"health", "version", "config-path", "init-db",
|
||
"audit-syllabus", "batch-audit-syllabus",
|
||
"render-html", "sync-norm-library",
|
||
}
|
||
missing = expected - set(COMMANDS.keys())
|
||
if missing:
|
||
return False, f"COMMANDS 缺入口：{sorted(missing)}"
|
||
return True, f"8 入口齐全（{len(expected)}/{len(COMMANDS)}）"
|
||
|
||
|
||
def test_cli_health_returns_zero() -> tuple[bool, str]:
|
||
"""`python scripts/main.py health` 应返回退出码 0"""
|
||
import subprocess
|
||
|
||
proc = subprocess.run(
|
||
[sys.executable, str(_SCRIPTS_DIR / "main.py"), "health"],
|
||
capture_output=True, text=True, timeout=15,
|
||
)
|
||
if proc.returncode != 0:
|
||
return False, (
|
||
f"exit={proc.returncode} stderr={proc.stderr.strip()[:200]} "
|
||
f"stdout={proc.stdout.strip()[:200]}"
|
||
)
|
||
return True, "exit=0"
|
||
|
||
|
||
def test_cli_version_prints_skill_version() -> tuple[bool, str]:
|
||
"""`python scripts/main.py version` 应在 stdout 输出 SKILL_VERSION"""
|
||
import subprocess
|
||
|
||
from util.constants import SKILL_VERSION
|
||
|
||
proc = subprocess.run(
|
||
[sys.executable, str(_SCRIPTS_DIR / "main.py"), "version"],
|
||
capture_output=True, text=True, timeout=15,
|
||
)
|
||
out = proc.stdout + proc.stderr
|
||
if proc.returncode != 0:
|
||
return False, f"exit={proc.returncode} out={out[:200]}"
|
||
if SKILL_VERSION not in out:
|
||
return False, f"stdout 未含 SKILL_VERSION={SKILL_VERSION}，实测={out.strip()[:200]}"
|
||
return True, f"stdout 含 SKILL_VERSION={SKILL_VERSION}"
|
||
|
||
|
||
def test_cli_unknown_command_returns_nonzero() -> tuple[bool, str]:
|
||
"""未知命令应返回非 0 退出码 + 列出可用命令"""
|
||
import subprocess
|
||
|
||
proc = subprocess.run(
|
||
[sys.executable, str(_SCRIPTS_DIR / "main.py"), "no-such-cmd"],
|
||
capture_output=True, text=True, timeout=15,
|
||
)
|
||
if proc.returncode == 0:
|
||
return False, "未知命令不应返回 0"
|
||
out = proc.stdout + proc.stderr
|
||
if "health" not in out and "audit-syllabus" not in out:
|
||
return False, f"未知命令应打印可用命令列表，实测={out[:200]}"
|
||
return True, "未知命令 → 非 0 + 列出可用命令"
|
||
|
||
|
||
# ---------- 入口 ----------
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description="默认 release 测试套件")
|
||
parser.add_argument("-v", "--verbose", action="store_true",
|
||
help="详细输出每个用例结果")
|
||
args = parser.parse_args()
|
||
|
||
global _VERBOSE
|
||
_VERBOSE = args.verbose
|
||
|
||
print("=== check-course-syllabus-aa release tests ===", flush=True)
|
||
|
||
_run("constants/skill_md_version_alignment", test_skill_md_version_alignment)
|
||
_run("constants/skill_md_slug_alignment", test_skill_md_slug_alignment)
|
||
_run("constants/threshold_defaults", test_constants_threshold_defaults)
|
||
_run("util/clamp_score", test_clamp_score)
|
||
_run("util/weighted_total_default", test_weighted_total_default)
|
||
_run("util/weighted_total_overrides", test_weighted_total_overrides)
|
||
_run("util/weighted_total_clamps_overshoot", test_weighted_total_clamps_overshoot)
|
||
_run("util/classify_audit_status_pass", test_classify_audit_status_pass)
|
||
_run("util/classify_audit_status_modify", test_classify_audit_status_modify)
|
||
_run("util/classify_audit_status_rewrite", test_classify_audit_status_rewrite)
|
||
_run("util/classify_audit_status_swapped", test_classify_audit_status_swapped_thresholds)
|
||
_run("util/parse_score_threshold", test_parse_score_threshold)
|
||
_run("util/parse_ideology_keywords_default", test_parse_ideology_keywords_default)
|
||
_run("util/parse_ideology_keywords_json", test_parse_ideology_keywords_json)
|
||
_run("util/parse_ideology_keywords_invalid", test_parse_ideology_keywords_invalid)
|
||
_run("util/count_keyword_hits_basic", test_count_keyword_hits_basic)
|
||
_run("util/count_keyword_hits_repeats", test_count_keyword_hits_repeats)
|
||
_run("util/count_keyword_hits_edge_cases", test_count_keyword_hits_edge_cases)
|
||
_run("cli/command_set", test_cli_command_set)
|
||
_run("cli/health_returns_zero", test_cli_health_returns_zero)
|
||
_run("cli/version_prints_skill_version", test_cli_version_prints_skill_version)
|
||
_run("cli/unknown_command_nonzero", test_cli_unknown_command_returns_nonzero)
|
||
|
||
total = len(_RESULTS)
|
||
passed = sum(1 for _, ok, _ in _RESULTS if ok)
|
||
failed = total - passed
|
||
|
||
print()
|
||
print(f"=== Summary: {passed}/{total} passed, {failed} failed ===", flush=True)
|
||
if failed:
|
||
print("\nFailures:")
|
||
for name, ok, msg in _RESULTS:
|
||
if not ok:
|
||
print(f" - {name}: {msg}")
|
||
return 1
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main()) |