266 lines
9.6 KiB
Python
266 lines
9.6 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import pathlib
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
|
|
SKILL_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]
|
|
SCRIPT = SKILL_ROOT / "scripts" / "lifecycle_status_guard_scan.py"
|
|
TMP_ROOT = REPO_ROOT / "tmp" / "lifecycle-status-guard-scan-tests"
|
|
|
|
|
|
def run_cli(*args: str) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(
|
|
[sys.executable, str(SCRIPT), *args],
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
|
|
|
|
def write_text(path: pathlib.Path, text: str) -> pathlib.Path:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(text, encoding="utf-8")
|
|
return path
|
|
|
|
|
|
def write_config(path: pathlib.Path, config: dict[str, object]) -> pathlib.Path:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(config, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
return path
|
|
|
|
|
|
def load_report(output_dir: pathlib.Path) -> dict[str, object]:
|
|
return json.loads((output_dir / "lifecycle-status-guard-scan.json").read_text(encoding="utf-8"))
|
|
|
|
|
|
class LifecycleStatusGuardScanTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
TMP_ROOT.mkdir(parents=True, exist_ok=True)
|
|
self.tmp_dir = tempfile.TemporaryDirectory(dir=TMP_ROOT)
|
|
self.root = pathlib.Path(self.tmp_dir.name) / "scan-root"
|
|
self.output_dir = pathlib.Path(self.tmp_dir.name) / "output"
|
|
self.config_path = pathlib.Path(self.tmp_dir.name) / "config.json"
|
|
self.root.mkdir()
|
|
|
|
def tearDown(self) -> None:
|
|
self.tmp_dir.cleanup()
|
|
|
|
@classmethod
|
|
def tearDownClass(cls) -> None:
|
|
if TMP_ROOT.exists():
|
|
shutil.rmtree(TMP_ROOT)
|
|
|
|
def run_scan(self, config: dict[str, object]) -> tuple[subprocess.CompletedProcess[str], dict[str, object]]:
|
|
write_config(self.config_path, config)
|
|
result = run_cli(
|
|
"--scan-root",
|
|
str(self.root),
|
|
"--config",
|
|
str(self.config_path),
|
|
"--output-dir",
|
|
str(self.output_dir),
|
|
)
|
|
return result, load_report(self.output_dir)
|
|
|
|
def test_markdown_front_matter_stable_without_evidence_is_blocking(self) -> None:
|
|
write_text(
|
|
self.root / "model-card.md",
|
|
"---\nstatus: stable\nname: QPI\n---\n\nEngineering checks passed.\n",
|
|
)
|
|
|
|
result, report = self.run_scan(
|
|
{
|
|
"watched_paths": ["**/*.md"],
|
|
"status_fields": ["status"],
|
|
"forbidden_status_values": ["stable", "accepted"],
|
|
"required_evidence_markers": ["owner_decision", "ccra_review"],
|
|
}
|
|
)
|
|
|
|
self.assertEqual(result.returncode, 1, result.stderr)
|
|
self.assertEqual(report["machine_readable_summary"]["blocking_count"], 1)
|
|
finding = report["field_level_findings"][0]
|
|
self.assertEqual(finding["field"], "status")
|
|
self.assertEqual(finding["value"], "stable")
|
|
self.assertFalse(finding["evidence_present"])
|
|
|
|
def test_draft_status_is_allowed(self) -> None:
|
|
write_text(self.root / "model-card.md", "---\nstatus: draft\n---\n\nStill in draft.\n")
|
|
|
|
result, report = self.run_scan(
|
|
{
|
|
"watched_paths": ["**/*.md"],
|
|
"status_fields": ["status"],
|
|
"forbidden_status_values": ["stable", "accepted"],
|
|
"warning_status_values": ["candidate"],
|
|
"required_evidence_markers": ["owner_decision"],
|
|
}
|
|
)
|
|
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(report["field_level_findings"], [])
|
|
self.assertEqual(report["blocking_errors"], [])
|
|
|
|
def test_owner_approved_phrase_without_nearby_evidence_is_blocking(self) -> None:
|
|
write_text(self.root / "handoff.md", "The model is owner-approved for stable use.\n")
|
|
|
|
result, report = self.run_scan(
|
|
{
|
|
"watched_paths": ["**/*.md"],
|
|
"approved_phrases": ["owner-approved"],
|
|
"required_evidence_markers": ["owner decision", "owner_decision"],
|
|
}
|
|
)
|
|
|
|
self.assertEqual(result.returncode, 1, result.stderr)
|
|
finding = report["phrase_level_findings"][0]
|
|
self.assertEqual(finding["phrase"], "owner-approved")
|
|
self.assertEqual(finding["severity"], "blocking")
|
|
self.assertFalse(finding["evidence_present"])
|
|
|
|
def test_owner_approved_phrase_with_nearby_evidence_is_observation_only(self) -> None:
|
|
write_text(
|
|
self.root / "handoff.md",
|
|
"The model is owner-approved. Owner decision: accepted in review note 2026-06-19.\n",
|
|
)
|
|
|
|
result, report = self.run_scan(
|
|
{
|
|
"watched_paths": ["**/*.md"],
|
|
"approved_phrases": ["owner-approved"],
|
|
"required_evidence_markers": ["owner decision"],
|
|
}
|
|
)
|
|
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(report["blocking_errors"], [])
|
|
self.assertEqual(report["observations"][0]["kind"], "evidence_present_claim")
|
|
|
|
def test_ccra_approved_phrase_without_review_reference_is_blocking(self) -> None:
|
|
write_text(self.root / "review.md", "Local output says ccra-approved and final.\n")
|
|
|
|
result, report = self.run_scan(
|
|
{
|
|
"watched_paths": ["**/*.md"],
|
|
"approved_phrases": ["ccra-approved"],
|
|
"required_evidence_markers": ["ccra review", "web ccra"],
|
|
}
|
|
)
|
|
|
|
self.assertEqual(result.returncode, 1, result.stderr)
|
|
self.assertEqual(report["phrase_level_findings"][0]["phrase"], "ccra-approved")
|
|
|
|
def test_json_status_evidence_must_be_in_same_object(self) -> None:
|
|
write_text(
|
|
self.root / "models.json",
|
|
json.dumps(
|
|
{
|
|
"models": [
|
|
{"name": "qpi", "status": "stable"},
|
|
{"name": "other", "owner_decision": "approved elsewhere"},
|
|
]
|
|
},
|
|
ensure_ascii=False,
|
|
),
|
|
)
|
|
|
|
result, report = self.run_scan(
|
|
{
|
|
"watched_paths": ["**/*.json"],
|
|
"status_fields": ["status"],
|
|
"forbidden_status_values": ["stable"],
|
|
"required_evidence_markers": ["owner_decision"],
|
|
}
|
|
)
|
|
|
|
self.assertEqual(result.returncode, 1, result.stderr)
|
|
finding = report["field_level_findings"][0]
|
|
self.assertEqual(finding["source"], "structured")
|
|
self.assertEqual(finding["path"], "models[0].status")
|
|
self.assertFalse(finding["evidence_present"])
|
|
|
|
def test_json_status_with_evidence_in_same_object_is_observation_only(self) -> None:
|
|
write_text(
|
|
self.root / "models.json",
|
|
json.dumps(
|
|
{"model": {"status": "stable", "owner_decision": "owner accepted"}},
|
|
ensure_ascii=False,
|
|
),
|
|
)
|
|
|
|
result, report = self.run_scan(
|
|
{
|
|
"watched_paths": ["**/*.json"],
|
|
"status_fields": ["status"],
|
|
"forbidden_status_values": ["stable"],
|
|
"required_evidence_markers": ["owner_decision"],
|
|
}
|
|
)
|
|
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(report["blocking_errors"], [])
|
|
self.assertEqual(report["observations"][0]["kind"], "field_evidence_present")
|
|
|
|
def test_allowed_context_exempts_phrase_but_not_structured_status_field(self) -> None:
|
|
write_text(
|
|
self.root / "policy.md",
|
|
"---\nstatus: stable\n---\n\nPolicy: do not say owner-approved without a decision.\n",
|
|
)
|
|
|
|
result, report = self.run_scan(
|
|
{
|
|
"watched_paths": ["**/*.md"],
|
|
"status_fields": ["status"],
|
|
"forbidden_status_values": ["stable"],
|
|
"approved_phrases": ["owner-approved"],
|
|
"allowed_contexts": ["do not say"],
|
|
"required_evidence_markers": ["owner decision"],
|
|
}
|
|
)
|
|
|
|
self.assertEqual(result.returncode, 1, result.stderr)
|
|
self.assertEqual(len(report["field_level_findings"]), 1)
|
|
self.assertEqual(report["phrase_level_findings"], [])
|
|
|
|
def test_chinese_approval_phrase_and_utf8_filename(self) -> None:
|
|
write_text(self.root / "模型状态.md", "本轮已经获得所有者批准，可以进入稳定状态。\n")
|
|
|
|
result, report = self.run_scan(
|
|
{
|
|
"watched_paths": ["**/*.md"],
|
|
"approved_phrases": ["所有者批准"],
|
|
"required_evidence_markers": ["所有者决定记录"],
|
|
}
|
|
)
|
|
|
|
self.assertEqual(result.returncode, 1, result.stderr)
|
|
finding = report["phrase_level_findings"][0]
|
|
self.assertEqual(finding["phrase"], "所有者批准")
|
|
self.assertIn("模型状态.md", finding["file"])
|
|
|
|
def test_warning_status_value_does_not_fail_scan(self) -> None:
|
|
write_text(self.root / "model.yml", "lifecycle: candidate\n")
|
|
|
|
result, report = self.run_scan(
|
|
{
|
|
"watched_paths": ["**/*.yml"],
|
|
"status_fields": ["lifecycle"],
|
|
"warning_status_values": ["candidate"],
|
|
}
|
|
)
|
|
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(report["warnings"][0]["severity"], "warning")
|
|
self.assertEqual(report["blocking_errors"], [])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|