Compare commits
No commits in common. "7e19041048094064716bf5caaacc25a602d9219a" and "419d70ab948bb88dbbb004c6764ada7ce787bf96" have entirely different histories.
7e19041048
...
419d70ab94
|
|
@ -22,24 +22,14 @@ PowerShell:
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill bundle-zip
|
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill bundle-zip
|
||||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill lifecycle-status-guard-scan
|
|
||||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill repair-markdown-citations
|
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill repair-markdown-citations
|
||||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill regression-validation-gate-runner
|
|
||||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill review-bundle-audit
|
|
||||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill review-context-builder
|
|
||||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill routing-behavior-diff-audit
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Use `-Force` to back up and replace an existing installed copy:
|
Use `-Force` to back up and replace an existing installed copy:
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill bundle-zip -Force
|
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill bundle-zip -Force
|
||||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill lifecycle-status-guard-scan -Force
|
|
||||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill repair-markdown-citations -Force
|
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill repair-markdown-citations -Force
|
||||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill regression-validation-gate-runner -Force
|
|
||||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill review-bundle-audit -Force
|
|
||||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill review-context-builder -Force
|
|
||||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill routing-behavior-diff-audit -Force
|
|
||||||
```
|
```
|
||||||
|
|
||||||
`-All` installs only registry rows with an installable status and `default` install policy. Optional Skills must be named explicitly.
|
`-All` installs only registry rows with an installable status and `default` install policy. Optional Skills must be named explicitly.
|
||||||
|
|
|
||||||
|
|
@ -15,12 +15,7 @@ Existing CCPE content is not migrated here.
|
||||||
| --- | --- | --- | --- | --- | --- | --- |
|
| --- | --- | --- | --- | --- | --- | --- |
|
||||||
| `bundle-zip` | Create zip archives from explicit file lists while preserving source-relative paths and validating entries by readback. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\bundle-zip` | `installed` | `default` | `C:\Users\wangq\.agents\skills\bundle-zip` | none |
|
| `bundle-zip` | Create zip archives from explicit file lists while preserving source-relative paths and validating entries by readback. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\bundle-zip` | `installed` | `default` | `C:\Users\wangq\.agents\skills\bundle-zip` | none |
|
||||||
| `fix-title` | Shift Markdown ATX heading depth for pasted LLM replies before insertion under parent sections. | `C:\Users\wangq\Documents\Codex\knowledge-vault\skills\fix-title` | `installed` | `default` | `C:\Users\wangq\.agents\skills\fix-title` | none |
|
| `fix-title` | Shift Markdown ATX heading depth for pasted LLM replies before insertion under parent sections. | `C:\Users\wangq\Documents\Codex\knowledge-vault\skills\fix-title` | `installed` | `default` | `C:\Users\wangq\.agents\skills\fix-title` | none |
|
||||||
| `lifecycle-status-guard-scan` | Scan configured Markdown, JSON, YAML, and text files for lifecycle/status overclaim candidates and missing local evidence markers. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\lifecycle-status-guard-scan` | `installed` | `default` | `C:\Users\wangq\.agents\skills\lifecycle-status-guard-scan` | none |
|
|
||||||
| `repair-markdown-citations` | Repair ChatGPT/Deep Research Markdown citation tokens into standard footnotes and a deduplicated reference section using exact turn-ID metadata mapping. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\repair-markdown-citations` | `installed` | `default` | `C:\Users\wangq\.agents\skills\repair-markdown-citations` | none |
|
| `repair-markdown-citations` | Repair ChatGPT/Deep Research Markdown citation tokens into standard footnotes and a deduplicated reference section using exact turn-ID metadata mapping. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\repair-markdown-citations` | `installed` | `default` | `C:\Users\wangq\.agents\skills\repair-markdown-citations` | none |
|
||||||
| `regression-validation-gate-runner` | Dry-run or execute project-declared regression and validation gates from a manifest while capturing logs, exit codes, durations, skipped gates, and changed-file notes. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\regression-validation-gate-runner` | `installed` | `default` | `C:\Users\wangq\.agents\skills\regression-validation-gate-runner` | none |
|
|
||||||
| `review-bundle-audit` | Preflight review, handoff, release, CCRA, Web upload, or code-review bundle directories for required files, manifests, sidecars, reports, zip readability, warnings, and blockers. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\review-bundle-audit` | `installed` | `default` | `C:\Users\wangq\.agents\skills\review-bundle-audit` | none |
|
|
||||||
| `review-context-builder` | Build file-first review, audit, planning, release, handoff, PR, or Agent invocation context indexes and manifests from configured local source roots, patterns, and metadata. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\review-context-builder` | `installed` | `default` | `C:\Users\wangq\.agents\skills\review-context-builder` | none |
|
|
||||||
| `routing-behavior-diff-audit` | Compare before/after selector, classifier, or routing outputs for targeted changes, collateral non-target changes, no-call changes, missing cases, and expected-route mismatch changes. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\routing-behavior-diff-audit` | `installed` | `default` | `C:\Users\wangq\.agents\skills\routing-behavior-diff-audit` | none |
|
|
||||||
| `voice-generation` | Generate spoken audio from Markdown scripts using the local `voice-gen` CLI and MiniMax `mmx`, including custom voice clone registration and voice registry management. | `D:\AI\ClaudeCode\voice-ge` | `installed` | `conditional` | `C:\Users\wangq\.agents\skills\voice-generation` | none |
|
| `voice-generation` | Generate spoken audio from Markdown scripts using the local `voice-gen` CLI and MiniMax `mmx`, including custom voice clone registration and voice registry management. | `D:\AI\ClaudeCode\voice-ge` | `installed` | `conditional` | `C:\Users\wangq\.agents\skills\voice-generation` | none |
|
||||||
|
|
||||||
## Status Values
|
## Status Values
|
||||||
|
|
|
||||||
|
|
@ -1,41 +0,0 @@
|
||||||
# Lifecycle Status Guard Scan
|
|
||||||
|
|
||||||
Source Skill for scanning lifecycle/status overclaim candidates in Markdown, JSON, YAML, and text files.
|
|
||||||
|
|
||||||
The canonical entry point is `SKILL.md`; the deterministic scanner is in `scripts/lifecycle_status_guard_scan.py`.
|
|
||||||
|
|
||||||
## Original Source
|
|
||||||
|
|
||||||
```text
|
|
||||||
Original source: C:\Users\wangq\Documents\Codex\ccpe-system\requirements\skills-vault\2026-06-19-lifecycle-status-guard-scan.md
|
|
||||||
Migration date: 2026-06-19
|
|
||||||
Migration status: first public Skill implementation
|
|
||||||
Behavior preserved: deterministic scan only; no lifecycle judgment or file edits
|
|
||||||
Known gaps: no project-specific policy preset beyond the example fixture
|
|
||||||
```
|
|
||||||
|
|
||||||
## Layout
|
|
||||||
|
|
||||||
```text
|
|
||||||
SKILL.md
|
|
||||||
README.md
|
|
||||||
agents/openai.yaml
|
|
||||||
fixtures/lifecycle-guard-config.example.yaml
|
|
||||||
scripts/lifecycle_status_guard_scan.py
|
|
||||||
tests/test_lifecycle_status_guard_scan.py
|
|
||||||
```
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
conda run -n skills-vault python .\scripts\lifecycle_status_guard_scan.py `
|
|
||||||
--scan-root C:\path\project `
|
|
||||||
--config C:\path\lifecycle-guard-config.yaml `
|
|
||||||
--output-dir C:\path\helper-outputs
|
|
||||||
```
|
|
||||||
|
|
||||||
## Tests
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
conda run -n skills-vault python -B -m unittest discover -s skills/lifecycle-status-guard-scan/tests -v
|
|
||||||
```
|
|
||||||
|
|
@ -1,102 +0,0 @@
|
||||||
---
|
|
||||||
name: lifecycle-status-guard-scan
|
|
||||||
description: Use when Codex needs to scan Markdown, JSON, YAML, or text files for lifecycle/status overclaims such as stable, accepted, owner-approved, CCRA-approved, released, production-ready, final, or equivalent Chinese phrasing before review, release, handoff, or governance work.
|
|
||||||
---
|
|
||||||
|
|
||||||
# Lifecycle Status Guard Scan
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
Run a deterministic, configurable scan for lifecycle/status claim candidates. Use the bundled script to produce Markdown and JSON reports that a human, Owner, CCRA reviewer, or upper-layer agent can judge.
|
|
||||||
|
|
||||||
The scanner is evidence-producing only. It must not edit status fields, promote or downgrade lifecycle state, or decide whether an approval claim is true.
|
|
||||||
|
|
||||||
## Command
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
conda run -n skills-vault python .\scripts\lifecycle_status_guard_scan.py `
|
|
||||||
--scan-root C:\path\project `
|
|
||||||
--config C:\path\lifecycle-guard-config.yaml `
|
|
||||||
--output-dir C:\path\helper-outputs
|
|
||||||
```
|
|
||||||
|
|
||||||
When installed under `C:\Users\wangq\.agents\skills\lifecycle-status-guard-scan`, run from that Skill directory or use the installed script path.
|
|
||||||
|
|
||||||
## Config
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
watched_paths:
|
|
||||||
- "**/*.md"
|
|
||||||
- "**/*.json"
|
|
||||||
- "**/*.yaml"
|
|
||||||
status_fields:
|
|
||||||
- status
|
|
||||||
- lifecycle
|
|
||||||
forbidden_status_values:
|
|
||||||
- stable
|
|
||||||
- accepted
|
|
||||||
warning_status_values:
|
|
||||||
- candidate
|
|
||||||
required_evidence_markers:
|
|
||||||
- owner_decision
|
|
||||||
- ccra_review
|
|
||||||
approved_phrases:
|
|
||||||
- owner-approved
|
|
||||||
- ccra-approved
|
|
||||||
- 所有者批准
|
|
||||||
forbidden_phrases:
|
|
||||||
- production-ready
|
|
||||||
allowed_contexts:
|
|
||||||
- do not say
|
|
||||||
- example:
|
|
||||||
```
|
|
||||||
|
|
||||||
Config files may be JSON, YAML, or YML.
|
|
||||||
|
|
||||||
## Rules
|
|
||||||
|
|
||||||
- Treat `approved_phrases` as approval-claim phrases requiring evidence. They are not a whitelist and do not mean the claim is accepted.
|
|
||||||
- If an approval claim phrase lacks a nearby `required_evidence_markers` match, report a blocking phrase finding.
|
|
||||||
- If an approval claim phrase has nearby evidence, report only an evidence-present observation. Do not judge whether the approval is real.
|
|
||||||
- For structured fields, associate evidence only within the same front matter block or JSON/YAML object that contains the matched field.
|
|
||||||
- For phrase claims, associate evidence only within the configured nearby text window, default 240 characters.
|
|
||||||
- Apply `allowed_contexts` only to phrase-level claims, such as negations, examples, or policy text. Never use allowed contexts to exempt structured fields like `status: stable`.
|
|
||||||
|
|
||||||
## Outputs
|
|
||||||
|
|
||||||
The script writes:
|
|
||||||
|
|
||||||
```text
|
|
||||||
lifecycle-status-guard-scan.md
|
|
||||||
lifecycle-status-guard-scan.json
|
|
||||||
```
|
|
||||||
|
|
||||||
Reports include:
|
|
||||||
|
|
||||||
- files scanned
|
|
||||||
- possible overclaims
|
|
||||||
- field-level findings
|
|
||||||
- phrase-level findings
|
|
||||||
- missing evidence markers
|
|
||||||
- warnings
|
|
||||||
- blocking errors
|
|
||||||
- evidence-present observations
|
|
||||||
- machine-readable summary
|
|
||||||
|
|
||||||
Exit code is `1` when blocking findings exist and `0` when the scan has only warnings or observations.
|
|
||||||
|
|
||||||
## Safety
|
|
||||||
|
|
||||||
- Read only configured files under `scan_root`.
|
|
||||||
- Write only the two report files under `output_dir`.
|
|
||||||
- Do not modify scanned files.
|
|
||||||
- Do not treat evidence markers as proof of approval; they only prevent a missing-evidence blocking result.
|
|
||||||
|
|
||||||
## Validation
|
|
||||||
|
|
||||||
After changing scanner behavior, run:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
conda run -n skills-vault python -B -m unittest discover -s skills/lifecycle-status-guard-scan/tests -v
|
|
||||||
conda run -n skills-vault python -B C:\Users\wangq\.codex\skills\.system\skill-creator\scripts\quick_validate.py skills/lifecycle-status-guard-scan
|
|
||||||
```
|
|
||||||
|
|
@ -1,4 +0,0 @@
|
||||||
interface:
|
|
||||||
display_name: "Lifecycle Status Guard Scan"
|
|
||||||
short_description: "Scan lifecycle claims for evidence gaps"
|
|
||||||
default_prompt: "Use $lifecycle-status-guard-scan to scan configured project files for lifecycle or approval claims that need evidence."
|
|
||||||
|
|
@ -1,42 +0,0 @@
|
||||||
watched_paths:
|
|
||||||
- "**/*.md"
|
|
||||||
- "**/*.json"
|
|
||||||
- "**/*.yaml"
|
|
||||||
- "**/*.yml"
|
|
||||||
status_fields:
|
|
||||||
- status
|
|
||||||
- lifecycle
|
|
||||||
- model.status
|
|
||||||
- model.lifecycle
|
|
||||||
forbidden_status_values:
|
|
||||||
- stable
|
|
||||||
- accepted
|
|
||||||
- owner-approved
|
|
||||||
- ccra-approved
|
|
||||||
warning_status_values:
|
|
||||||
- candidate
|
|
||||||
- ready-for-review
|
|
||||||
required_evidence_markers:
|
|
||||||
- owner_decision
|
|
||||||
- owner decision
|
|
||||||
- ccra_review
|
|
||||||
- ccra review
|
|
||||||
- web ccra
|
|
||||||
- 所有者决定记录
|
|
||||||
approved_phrases:
|
|
||||||
- owner-approved
|
|
||||||
- ccra-approved
|
|
||||||
- Owner approved
|
|
||||||
- CCRA approved
|
|
||||||
- 所有者批准
|
|
||||||
- CCRA批准
|
|
||||||
forbidden_phrases:
|
|
||||||
- production-ready
|
|
||||||
- released to production
|
|
||||||
- final approved
|
|
||||||
allowed_contexts:
|
|
||||||
- do not say
|
|
||||||
- must not claim
|
|
||||||
- example:
|
|
||||||
- 示例:
|
|
||||||
evidence_window_chars: 240
|
|
||||||
|
|
@ -1,448 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import fnmatch
|
|
||||||
import json
|
|
||||||
import pathlib
|
|
||||||
import re
|
|
||||||
import sys
|
|
||||||
from collections.abc import Iterable, Mapping
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import yaml
|
|
||||||
|
|
||||||
|
|
||||||
REPORT_BASENAME = "lifecycle-status-guard-scan"
|
|
||||||
DEFAULT_EVIDENCE_WINDOW_CHARS = 240
|
|
||||||
|
|
||||||
|
|
||||||
def as_list(value: Any) -> list[str]:
|
|
||||||
if value is None:
|
|
||||||
return []
|
|
||||||
if isinstance(value, str):
|
|
||||||
return [value]
|
|
||||||
if isinstance(value, Iterable):
|
|
||||||
return [str(item) for item in value if item is not None]
|
|
||||||
return [str(value)]
|
|
||||||
|
|
||||||
|
|
||||||
def lower_set(values: Iterable[str]) -> set[str]:
|
|
||||||
return {value.casefold() for value in values}
|
|
||||||
|
|
||||||
|
|
||||||
def normalize_config(raw_config: Mapping[str, Any]) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"watched_paths": as_list(raw_config.get("watched_paths")) or ["**/*"],
|
|
||||||
"status_fields": as_list(raw_config.get("status_fields")),
|
|
||||||
"forbidden_status_values": lower_set(as_list(raw_config.get("forbidden_status_values"))),
|
|
||||||
"warning_status_values": lower_set(as_list(raw_config.get("warning_status_values"))),
|
|
||||||
"required_evidence_markers": as_list(raw_config.get("required_evidence_markers")),
|
|
||||||
"approved_phrases": as_list(raw_config.get("approved_phrases")),
|
|
||||||
"forbidden_phrases": as_list(raw_config.get("forbidden_phrases")),
|
|
||||||
"allowed_contexts": as_list(raw_config.get("allowed_contexts")),
|
|
||||||
"evidence_window_chars": int(
|
|
||||||
raw_config.get("evidence_window_chars", DEFAULT_EVIDENCE_WINDOW_CHARS)
|
|
||||||
),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def load_config(path: pathlib.Path) -> dict[str, Any]:
|
|
||||||
text = path.read_text(encoding="utf-8")
|
|
||||||
if path.suffix.lower() == ".json":
|
|
||||||
parsed = json.loads(text)
|
|
||||||
else:
|
|
||||||
parsed = yaml.safe_load(text)
|
|
||||||
if not isinstance(parsed, Mapping):
|
|
||||||
raise ValueError("Config must be a JSON or YAML object.")
|
|
||||||
return normalize_config(parsed)
|
|
||||||
|
|
||||||
|
|
||||||
def rel_path(path: pathlib.Path, root: pathlib.Path) -> str:
|
|
||||||
return path.relative_to(root).as_posix()
|
|
||||||
|
|
||||||
|
|
||||||
def discover_files(scan_root: pathlib.Path, watched_paths: list[str]) -> list[pathlib.Path]:
|
|
||||||
files: dict[pathlib.Path, None] = {}
|
|
||||||
for pattern in watched_paths:
|
|
||||||
for candidate in scan_root.glob(pattern):
|
|
||||||
if candidate.is_file():
|
|
||||||
files[candidate.resolve()] = None
|
|
||||||
return sorted(files)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_markdown_front_matter(text: str) -> Any | None:
|
|
||||||
if not text.startswith("---"):
|
|
||||||
return None
|
|
||||||
lines = text.splitlines()
|
|
||||||
if not lines or lines[0].strip() != "---":
|
|
||||||
return None
|
|
||||||
for index in range(1, len(lines)):
|
|
||||||
if lines[index].strip() == "---":
|
|
||||||
front_matter = "\n".join(lines[1:index])
|
|
||||||
parsed = yaml.safe_load(front_matter) if front_matter.strip() else {}
|
|
||||||
return parsed if isinstance(parsed, Mapping) else None
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def parse_structured_content(path: pathlib.Path, text: str) -> Any | None:
|
|
||||||
suffix = path.suffix.lower()
|
|
||||||
try:
|
|
||||||
if suffix == ".json":
|
|
||||||
return json.loads(text)
|
|
||||||
if suffix in {".yaml", ".yml"}:
|
|
||||||
return yaml.safe_load(text)
|
|
||||||
if suffix in {".md", ".markdown"}:
|
|
||||||
return parse_markdown_front_matter(text)
|
|
||||||
except Exception:
|
|
||||||
return None
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def scalar_values(value: Any) -> list[str]:
|
|
||||||
if isinstance(value, (str, int, float, bool)):
|
|
||||||
return [str(value)]
|
|
||||||
if isinstance(value, list):
|
|
||||||
values: list[str] = []
|
|
||||||
for item in value:
|
|
||||||
values.extend(scalar_values(item))
|
|
||||||
return values
|
|
||||||
return []
|
|
||||||
|
|
||||||
|
|
||||||
def object_contains_marker(value: Any, markers: list[str]) -> bool:
|
|
||||||
if not markers:
|
|
||||||
return False
|
|
||||||
marker_needles = [marker.casefold() for marker in markers]
|
|
||||||
|
|
||||||
def walk(current: Any) -> bool:
|
|
||||||
if isinstance(current, Mapping):
|
|
||||||
for key, nested in current.items():
|
|
||||||
key_text = str(key).casefold()
|
|
||||||
if any(marker in key_text for marker in marker_needles):
|
|
||||||
return True
|
|
||||||
if walk(nested):
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
if isinstance(current, list):
|
|
||||||
return any(walk(item) for item in current)
|
|
||||||
text = str(current).casefold()
|
|
||||||
return any(marker in text for marker in marker_needles)
|
|
||||||
|
|
||||||
return walk(value)
|
|
||||||
|
|
||||||
|
|
||||||
def line_col(text: str, index: int) -> tuple[int, int]:
|
|
||||||
line = text.count("\n", 0, index) + 1
|
|
||||||
last_newline = text.rfind("\n", 0, index)
|
|
||||||
column = index + 1 if last_newline == -1 else index - last_newline
|
|
||||||
return line, column
|
|
||||||
|
|
||||||
|
|
||||||
def make_context(text: str, index: int, length: int, window: int) -> str:
|
|
||||||
start = max(0, index - window)
|
|
||||||
end = min(len(text), index + length + window)
|
|
||||||
return re.sub(r"\s+", " ", text[start:end]).strip()
|
|
||||||
|
|
||||||
|
|
||||||
def context_has_any(context: str, values: list[str]) -> bool:
|
|
||||||
folded = context.casefold()
|
|
||||||
return any(value.casefold() in folded for value in values)
|
|
||||||
|
|
||||||
|
|
||||||
def find_phrase_matches(text: str, phrase: str) -> Iterable[int]:
|
|
||||||
if not phrase:
|
|
||||||
return []
|
|
||||||
folded_text = text.casefold()
|
|
||||||
folded_phrase = phrase.casefold()
|
|
||||||
matches: list[int] = []
|
|
||||||
start = 0
|
|
||||||
while True:
|
|
||||||
index = folded_text.find(folded_phrase, start)
|
|
||||||
if index == -1:
|
|
||||||
break
|
|
||||||
matches.append(index)
|
|
||||||
start = index + max(len(folded_phrase), 1)
|
|
||||||
return matches
|
|
||||||
|
|
||||||
|
|
||||||
def path_matches(field_path: str, key: str, status_fields: list[str]) -> bool:
|
|
||||||
if not status_fields:
|
|
||||||
return False
|
|
||||||
candidates = {field_path, key}
|
|
||||||
return any(
|
|
||||||
field in candidates or fnmatch.fnmatch(field_path, field) or fnmatch.fnmatch(key, field)
|
|
||||||
for field in status_fields
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def child_path(parent_path: str, key: str) -> str:
|
|
||||||
return f"{parent_path}.{key}" if parent_path else key
|
|
||||||
|
|
||||||
|
|
||||||
def list_path(parent_path: str, index: int) -> str:
|
|
||||||
return f"{parent_path}[{index}]" if parent_path else f"[{index}]"
|
|
||||||
|
|
||||||
|
|
||||||
def scan_structured(
|
|
||||||
data: Any,
|
|
||||||
file_rel: str,
|
|
||||||
config: dict[str, Any],
|
|
||||||
report: dict[str, Any],
|
|
||||||
) -> None:
|
|
||||||
status_fields = config["status_fields"]
|
|
||||||
forbidden_values = config["forbidden_status_values"]
|
|
||||||
warning_values = config["warning_status_values"]
|
|
||||||
evidence_markers = config["required_evidence_markers"]
|
|
||||||
|
|
||||||
def walk(current: Any, path: str, parent: Any) -> None:
|
|
||||||
if isinstance(current, Mapping):
|
|
||||||
for key_obj, value in current.items():
|
|
||||||
key = str(key_obj)
|
|
||||||
current_path = child_path(path, key)
|
|
||||||
if path_matches(current_path, key, status_fields):
|
|
||||||
for scalar in scalar_values(value):
|
|
||||||
value_key = scalar.casefold()
|
|
||||||
if value_key in forbidden_values:
|
|
||||||
evidence_present = object_contains_marker(parent, evidence_markers)
|
|
||||||
record = {
|
|
||||||
"file": file_rel,
|
|
||||||
"source": "structured",
|
|
||||||
"path": current_path,
|
|
||||||
"field": key,
|
|
||||||
"value": scalar,
|
|
||||||
"severity": "blocking" if not evidence_present else "observation",
|
|
||||||
"evidence_present": evidence_present,
|
|
||||||
"message": (
|
|
||||||
"Forbidden lifecycle/status value lacks local evidence marker."
|
|
||||||
if not evidence_present
|
|
||||||
else "Lifecycle/status value has a local evidence marker; approval validity is not decided by this scan."
|
|
||||||
),
|
|
||||||
}
|
|
||||||
if evidence_present:
|
|
||||||
observation = dict(record)
|
|
||||||
observation["kind"] = "field_evidence_present"
|
|
||||||
report["observations"].append(observation)
|
|
||||||
else:
|
|
||||||
report["field_level_findings"].append(record)
|
|
||||||
report["blocking_errors"].append(record["message"])
|
|
||||||
report["missing_evidence_markers"].append(
|
|
||||||
{
|
|
||||||
"file": file_rel,
|
|
||||||
"path": current_path,
|
|
||||||
"required_evidence_markers": evidence_markers,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
elif value_key in warning_values:
|
|
||||||
report["warnings"].append(
|
|
||||||
{
|
|
||||||
"file": file_rel,
|
|
||||||
"source": "structured",
|
|
||||||
"path": current_path,
|
|
||||||
"field": key,
|
|
||||||
"value": scalar,
|
|
||||||
"severity": "warning",
|
|
||||||
"message": "Warning lifecycle/status value matched configured policy.",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
walk(value, current_path, value if isinstance(value, Mapping) else current)
|
|
||||||
elif isinstance(current, list):
|
|
||||||
for index, item in enumerate(current):
|
|
||||||
walk(item, list_path(path, index), item if isinstance(item, Mapping) else parent)
|
|
||||||
|
|
||||||
walk(data, "", data)
|
|
||||||
|
|
||||||
|
|
||||||
def scan_phrases(
|
|
||||||
text: str,
|
|
||||||
file_rel: str,
|
|
||||||
config: dict[str, Any],
|
|
||||||
report: dict[str, Any],
|
|
||||||
) -> None:
|
|
||||||
evidence_markers = config["required_evidence_markers"]
|
|
||||||
allowed_contexts = config["allowed_contexts"]
|
|
||||||
window = config["evidence_window_chars"]
|
|
||||||
|
|
||||||
for phrase in config["approved_phrases"]:
|
|
||||||
for index in find_phrase_matches(text, phrase):
|
|
||||||
context = make_context(text, index, len(phrase), window)
|
|
||||||
if context_has_any(context, allowed_contexts):
|
|
||||||
continue
|
|
||||||
evidence_present = context_has_any(context, evidence_markers)
|
|
||||||
line, column = line_col(text, index)
|
|
||||||
record = {
|
|
||||||
"file": file_rel,
|
|
||||||
"source": "text",
|
|
||||||
"phrase": phrase,
|
|
||||||
"line": line,
|
|
||||||
"column": column,
|
|
||||||
"severity": "blocking" if not evidence_present else "observation",
|
|
||||||
"evidence_present": evidence_present,
|
|
||||||
"context": context,
|
|
||||||
"message": (
|
|
||||||
"Approval claim phrase lacks nearby evidence marker."
|
|
||||||
if not evidence_present
|
|
||||||
else "Approval claim phrase has nearby evidence marker; approval validity is not decided by this scan."
|
|
||||||
),
|
|
||||||
}
|
|
||||||
if evidence_present:
|
|
||||||
observation = dict(record)
|
|
||||||
observation["kind"] = "evidence_present_claim"
|
|
||||||
report["observations"].append(observation)
|
|
||||||
else:
|
|
||||||
report["phrase_level_findings"].append(record)
|
|
||||||
report["blocking_errors"].append(record["message"])
|
|
||||||
report["missing_evidence_markers"].append(
|
|
||||||
{
|
|
||||||
"file": file_rel,
|
|
||||||
"line": line,
|
|
||||||
"phrase": phrase,
|
|
||||||
"required_evidence_markers": evidence_markers,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
for phrase in config["forbidden_phrases"]:
|
|
||||||
for index in find_phrase_matches(text, phrase):
|
|
||||||
context = make_context(text, index, len(phrase), window)
|
|
||||||
if context_has_any(context, allowed_contexts):
|
|
||||||
continue
|
|
||||||
line, column = line_col(text, index)
|
|
||||||
record = {
|
|
||||||
"file": file_rel,
|
|
||||||
"source": "text",
|
|
||||||
"phrase": phrase,
|
|
||||||
"line": line,
|
|
||||||
"column": column,
|
|
||||||
"severity": "blocking",
|
|
||||||
"evidence_present": context_has_any(context, evidence_markers),
|
|
||||||
"context": context,
|
|
||||||
"message": "Forbidden phrase matched configured policy.",
|
|
||||||
}
|
|
||||||
report["phrase_level_findings"].append(record)
|
|
||||||
report["blocking_errors"].append(record["message"])
|
|
||||||
|
|
||||||
|
|
||||||
def empty_report(scan_root: pathlib.Path, config_path: pathlib.Path) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"scan_root": str(scan_root),
|
|
||||||
"config_path": str(config_path),
|
|
||||||
"files_scanned": [],
|
|
||||||
"possible_overclaims": [],
|
|
||||||
"field_level_findings": [],
|
|
||||||
"phrase_level_findings": [],
|
|
||||||
"missing_evidence_markers": [],
|
|
||||||
"warnings": [],
|
|
||||||
"blocking_errors": [],
|
|
||||||
"observations": [],
|
|
||||||
"machine_readable_summary": {},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def finalize_report(report: dict[str, Any]) -> None:
|
|
||||||
report["possible_overclaims"] = [
|
|
||||||
*report["field_level_findings"],
|
|
||||||
*report["phrase_level_findings"],
|
|
||||||
]
|
|
||||||
report["machine_readable_summary"] = {
|
|
||||||
"status": "FAIL" if report["blocking_errors"] else "PASS",
|
|
||||||
"files_scanned_count": len(report["files_scanned"]),
|
|
||||||
"field_finding_count": len(report["field_level_findings"]),
|
|
||||||
"phrase_finding_count": len(report["phrase_level_findings"]),
|
|
||||||
"missing_evidence_count": len(report["missing_evidence_markers"]),
|
|
||||||
"warning_count": len(report["warnings"]),
|
|
||||||
"blocking_count": len(report["blocking_errors"]),
|
|
||||||
"observation_count": len(report["observations"]),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def write_markdown_report(report: dict[str, Any], output_path: pathlib.Path) -> None:
|
|
||||||
summary = report["machine_readable_summary"]
|
|
||||||
lines = [
|
|
||||||
"# Lifecycle Status Guard Scan",
|
|
||||||
"",
|
|
||||||
"## Summary",
|
|
||||||
"",
|
|
||||||
f"- Status: `{summary['status']}`",
|
|
||||||
f"- Files scanned: {summary['files_scanned_count']}",
|
|
||||||
f"- Blocking findings: {summary['blocking_count']}",
|
|
||||||
f"- Warnings: {summary['warning_count']}",
|
|
||||||
f"- Evidence-present observations: {summary['observation_count']}",
|
|
||||||
"",
|
|
||||||
"## Blocking Findings",
|
|
||||||
"",
|
|
||||||
]
|
|
||||||
if report["possible_overclaims"]:
|
|
||||||
for finding in report["possible_overclaims"]:
|
|
||||||
location = finding.get("path") or f"line {finding.get('line')}"
|
|
||||||
claim = finding.get("value") or finding.get("phrase")
|
|
||||||
lines.append(f"- `{finding['file']}` {location}: `{claim}` - {finding['message']}")
|
|
||||||
else:
|
|
||||||
lines.append("- None")
|
|
||||||
lines.extend(["", "## Warnings", ""])
|
|
||||||
if report["warnings"]:
|
|
||||||
for warning in report["warnings"]:
|
|
||||||
lines.append(
|
|
||||||
f"- `{warning['file']}` {warning.get('path', '')}: `{warning.get('value', '')}` - {warning['message']}"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
lines.append("- None")
|
|
||||||
lines.extend(["", "## Observations", ""])
|
|
||||||
if report["observations"]:
|
|
||||||
for observation in report["observations"]:
|
|
||||||
location = observation.get("path") or f"line {observation.get('line')}"
|
|
||||||
claim = observation.get("value") or observation.get("phrase")
|
|
||||||
lines.append(
|
|
||||||
f"- `{observation['file']}` {location}: `{claim}` - evidence marker present; approval validity not decided."
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
lines.append("- None")
|
|
||||||
lines.extend(["", "## Files Scanned", ""])
|
|
||||||
for file_name in report["files_scanned"]:
|
|
||||||
lines.append(f"- `{file_name}`")
|
|
||||||
output_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
||||||
|
|
||||||
|
|
||||||
def run(scan_root: pathlib.Path, config_path: pathlib.Path, output_dir: pathlib.Path) -> int:
|
|
||||||
scan_root = scan_root.resolve()
|
|
||||||
config_path = config_path.resolve()
|
|
||||||
if not scan_root.is_dir():
|
|
||||||
raise ValueError(f"Scan root is not a directory: {scan_root}")
|
|
||||||
if not config_path.is_file():
|
|
||||||
raise ValueError(f"Config file does not exist: {config_path}")
|
|
||||||
|
|
||||||
config = load_config(config_path)
|
|
||||||
report = empty_report(scan_root, config_path)
|
|
||||||
|
|
||||||
for path in discover_files(scan_root, config["watched_paths"]):
|
|
||||||
file_rel = rel_path(path, scan_root)
|
|
||||||
text = path.read_text(encoding="utf-8", errors="replace")
|
|
||||||
report["files_scanned"].append(file_rel)
|
|
||||||
structured = parse_structured_content(path, text)
|
|
||||||
if isinstance(structured, (Mapping, list)):
|
|
||||||
scan_structured(structured, file_rel, config, report)
|
|
||||||
scan_phrases(text, file_rel, config, report)
|
|
||||||
|
|
||||||
finalize_report(report)
|
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
json_path = output_dir / f"{REPORT_BASENAME}.json"
|
|
||||||
md_path = output_dir / f"{REPORT_BASENAME}.md"
|
|
||||||
json_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
||||||
write_markdown_report(report, md_path)
|
|
||||||
return 1 if report["blocking_errors"] else 0
|
|
||||||
|
|
||||||
|
|
||||||
def main(argv: list[str] | None = None) -> int:
|
|
||||||
parser = argparse.ArgumentParser(description="Scan configured lifecycle/status claims.")
|
|
||||||
parser.add_argument("--scan-root", required=True, type=pathlib.Path)
|
|
||||||
parser.add_argument("--config", required=True, type=pathlib.Path)
|
|
||||||
parser.add_argument("--output-dir", required=True, type=pathlib.Path)
|
|
||||||
args = parser.parse_args(argv)
|
|
||||||
|
|
||||||
try:
|
|
||||||
return run(args.scan_root, args.config, args.output_dir)
|
|
||||||
except Exception as exc:
|
|
||||||
print(f"ERROR: {exc}", file=sys.stderr)
|
|
||||||
return 2
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
|
|
@ -1,265 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
import pathlib
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import tempfile
|
|
||||||
import unittest
|
|
||||||
|
|
||||||
|
|
||||||
SKILL_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
||||||
REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]
|
|
||||||
SCRIPT = SKILL_ROOT / "scripts" / "lifecycle_status_guard_scan.py"
|
|
||||||
TMP_ROOT = REPO_ROOT / "tmp" / "lifecycle-status-guard-scan-tests"
|
|
||||||
|
|
||||||
|
|
||||||
def run_cli(*args: str) -> subprocess.CompletedProcess[str]:
|
|
||||||
return subprocess.run(
|
|
||||||
[sys.executable, str(SCRIPT), *args],
|
|
||||||
text=True,
|
|
||||||
capture_output=True,
|
|
||||||
check=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def write_text(path: pathlib.Path, text: str) -> pathlib.Path:
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
path.write_text(text, encoding="utf-8")
|
|
||||||
return path
|
|
||||||
|
|
||||||
|
|
||||||
def write_config(path: pathlib.Path, config: dict[str, object]) -> pathlib.Path:
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
path.write_text(json.dumps(config, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
||||||
return path
|
|
||||||
|
|
||||||
|
|
||||||
def load_report(output_dir: pathlib.Path) -> dict[str, object]:
|
|
||||||
return json.loads((output_dir / "lifecycle-status-guard-scan.json").read_text(encoding="utf-8"))
|
|
||||||
|
|
||||||
|
|
||||||
class LifecycleStatusGuardScanTest(unittest.TestCase):
|
|
||||||
def setUp(self) -> None:
|
|
||||||
TMP_ROOT.mkdir(parents=True, exist_ok=True)
|
|
||||||
self.tmp_dir = tempfile.TemporaryDirectory(dir=TMP_ROOT)
|
|
||||||
self.root = pathlib.Path(self.tmp_dir.name) / "scan-root"
|
|
||||||
self.output_dir = pathlib.Path(self.tmp_dir.name) / "output"
|
|
||||||
self.config_path = pathlib.Path(self.tmp_dir.name) / "config.json"
|
|
||||||
self.root.mkdir()
|
|
||||||
|
|
||||||
def tearDown(self) -> None:
|
|
||||||
self.tmp_dir.cleanup()
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def tearDownClass(cls) -> None:
|
|
||||||
if TMP_ROOT.exists():
|
|
||||||
shutil.rmtree(TMP_ROOT)
|
|
||||||
|
|
||||||
def run_scan(self, config: dict[str, object]) -> tuple[subprocess.CompletedProcess[str], dict[str, object]]:
|
|
||||||
write_config(self.config_path, config)
|
|
||||||
result = run_cli(
|
|
||||||
"--scan-root",
|
|
||||||
str(self.root),
|
|
||||||
"--config",
|
|
||||||
str(self.config_path),
|
|
||||||
"--output-dir",
|
|
||||||
str(self.output_dir),
|
|
||||||
)
|
|
||||||
return result, load_report(self.output_dir)
|
|
||||||
|
|
||||||
def test_markdown_front_matter_stable_without_evidence_is_blocking(self) -> None:
|
|
||||||
write_text(
|
|
||||||
self.root / "model-card.md",
|
|
||||||
"---\nstatus: stable\nname: QPI\n---\n\nEngineering checks passed.\n",
|
|
||||||
)
|
|
||||||
|
|
||||||
result, report = self.run_scan(
|
|
||||||
{
|
|
||||||
"watched_paths": ["**/*.md"],
|
|
||||||
"status_fields": ["status"],
|
|
||||||
"forbidden_status_values": ["stable", "accepted"],
|
|
||||||
"required_evidence_markers": ["owner_decision", "ccra_review"],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 1, result.stderr)
|
|
||||||
self.assertEqual(report["machine_readable_summary"]["blocking_count"], 1)
|
|
||||||
finding = report["field_level_findings"][0]
|
|
||||||
self.assertEqual(finding["field"], "status")
|
|
||||||
self.assertEqual(finding["value"], "stable")
|
|
||||||
self.assertFalse(finding["evidence_present"])
|
|
||||||
|
|
||||||
def test_draft_status_is_allowed(self) -> None:
|
|
||||||
write_text(self.root / "model-card.md", "---\nstatus: draft\n---\n\nStill in draft.\n")
|
|
||||||
|
|
||||||
result, report = self.run_scan(
|
|
||||||
{
|
|
||||||
"watched_paths": ["**/*.md"],
|
|
||||||
"status_fields": ["status"],
|
|
||||||
"forbidden_status_values": ["stable", "accepted"],
|
|
||||||
"warning_status_values": ["candidate"],
|
|
||||||
"required_evidence_markers": ["owner_decision"],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual(report["field_level_findings"], [])
|
|
||||||
self.assertEqual(report["blocking_errors"], [])
|
|
||||||
|
|
||||||
def test_owner_approved_phrase_without_nearby_evidence_is_blocking(self) -> None:
|
|
||||||
write_text(self.root / "handoff.md", "The model is owner-approved for stable use.\n")
|
|
||||||
|
|
||||||
result, report = self.run_scan(
|
|
||||||
{
|
|
||||||
"watched_paths": ["**/*.md"],
|
|
||||||
"approved_phrases": ["owner-approved"],
|
|
||||||
"required_evidence_markers": ["owner decision", "owner_decision"],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 1, result.stderr)
|
|
||||||
finding = report["phrase_level_findings"][0]
|
|
||||||
self.assertEqual(finding["phrase"], "owner-approved")
|
|
||||||
self.assertEqual(finding["severity"], "blocking")
|
|
||||||
self.assertFalse(finding["evidence_present"])
|
|
||||||
|
|
||||||
def test_owner_approved_phrase_with_nearby_evidence_is_observation_only(self) -> None:
|
|
||||||
write_text(
|
|
||||||
self.root / "handoff.md",
|
|
||||||
"The model is owner-approved. Owner decision: accepted in review note 2026-06-19.\n",
|
|
||||||
)
|
|
||||||
|
|
||||||
result, report = self.run_scan(
|
|
||||||
{
|
|
||||||
"watched_paths": ["**/*.md"],
|
|
||||||
"approved_phrases": ["owner-approved"],
|
|
||||||
"required_evidence_markers": ["owner decision"],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual(report["blocking_errors"], [])
|
|
||||||
self.assertEqual(report["observations"][0]["kind"], "evidence_present_claim")
|
|
||||||
|
|
||||||
def test_ccra_approved_phrase_without_review_reference_is_blocking(self) -> None:
|
|
||||||
write_text(self.root / "review.md", "Local output says ccra-approved and final.\n")
|
|
||||||
|
|
||||||
result, report = self.run_scan(
|
|
||||||
{
|
|
||||||
"watched_paths": ["**/*.md"],
|
|
||||||
"approved_phrases": ["ccra-approved"],
|
|
||||||
"required_evidence_markers": ["ccra review", "web ccra"],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 1, result.stderr)
|
|
||||||
self.assertEqual(report["phrase_level_findings"][0]["phrase"], "ccra-approved")
|
|
||||||
|
|
||||||
def test_json_status_evidence_must_be_in_same_object(self) -> None:
|
|
||||||
write_text(
|
|
||||||
self.root / "models.json",
|
|
||||||
json.dumps(
|
|
||||||
{
|
|
||||||
"models": [
|
|
||||||
{"name": "qpi", "status": "stable"},
|
|
||||||
{"name": "other", "owner_decision": "approved elsewhere"},
|
|
||||||
]
|
|
||||||
},
|
|
||||||
ensure_ascii=False,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
result, report = self.run_scan(
|
|
||||||
{
|
|
||||||
"watched_paths": ["**/*.json"],
|
|
||||||
"status_fields": ["status"],
|
|
||||||
"forbidden_status_values": ["stable"],
|
|
||||||
"required_evidence_markers": ["owner_decision"],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 1, result.stderr)
|
|
||||||
finding = report["field_level_findings"][0]
|
|
||||||
self.assertEqual(finding["source"], "structured")
|
|
||||||
self.assertEqual(finding["path"], "models[0].status")
|
|
||||||
self.assertFalse(finding["evidence_present"])
|
|
||||||
|
|
||||||
def test_json_status_with_evidence_in_same_object_is_observation_only(self) -> None:
|
|
||||||
write_text(
|
|
||||||
self.root / "models.json",
|
|
||||||
json.dumps(
|
|
||||||
{"model": {"status": "stable", "owner_decision": "owner accepted"}},
|
|
||||||
ensure_ascii=False,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
result, report = self.run_scan(
|
|
||||||
{
|
|
||||||
"watched_paths": ["**/*.json"],
|
|
||||||
"status_fields": ["status"],
|
|
||||||
"forbidden_status_values": ["stable"],
|
|
||||||
"required_evidence_markers": ["owner_decision"],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual(report["blocking_errors"], [])
|
|
||||||
self.assertEqual(report["observations"][0]["kind"], "field_evidence_present")
|
|
||||||
|
|
||||||
def test_allowed_context_exempts_phrase_but_not_structured_status_field(self) -> None:
|
|
||||||
write_text(
|
|
||||||
self.root / "policy.md",
|
|
||||||
"---\nstatus: stable\n---\n\nPolicy: do not say owner-approved without a decision.\n",
|
|
||||||
)
|
|
||||||
|
|
||||||
result, report = self.run_scan(
|
|
||||||
{
|
|
||||||
"watched_paths": ["**/*.md"],
|
|
||||||
"status_fields": ["status"],
|
|
||||||
"forbidden_status_values": ["stable"],
|
|
||||||
"approved_phrases": ["owner-approved"],
|
|
||||||
"allowed_contexts": ["do not say"],
|
|
||||||
"required_evidence_markers": ["owner decision"],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 1, result.stderr)
|
|
||||||
self.assertEqual(len(report["field_level_findings"]), 1)
|
|
||||||
self.assertEqual(report["phrase_level_findings"], [])
|
|
||||||
|
|
||||||
def test_chinese_approval_phrase_and_utf8_filename(self) -> None:
|
|
||||||
write_text(self.root / "模型状态.md", "本轮已经获得所有者批准,可以进入稳定状态。\n")
|
|
||||||
|
|
||||||
result, report = self.run_scan(
|
|
||||||
{
|
|
||||||
"watched_paths": ["**/*.md"],
|
|
||||||
"approved_phrases": ["所有者批准"],
|
|
||||||
"required_evidence_markers": ["所有者决定记录"],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 1, result.stderr)
|
|
||||||
finding = report["phrase_level_findings"][0]
|
|
||||||
self.assertEqual(finding["phrase"], "所有者批准")
|
|
||||||
self.assertIn("模型状态.md", finding["file"])
|
|
||||||
|
|
||||||
def test_warning_status_value_does_not_fail_scan(self) -> None:
|
|
||||||
write_text(self.root / "model.yml", "lifecycle: candidate\n")
|
|
||||||
|
|
||||||
result, report = self.run_scan(
|
|
||||||
{
|
|
||||||
"watched_paths": ["**/*.yml"],
|
|
||||||
"status_fields": ["lifecycle"],
|
|
||||||
"warning_status_values": ["candidate"],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual(report["warnings"][0]["severity"], "warning")
|
|
||||||
self.assertEqual(report["blocking_errors"], [])
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
unittest.main()
|
|
||||||
|
|
@ -1,42 +0,0 @@
|
||||||
# Regression Validation Gate Runner
|
|
||||||
|
|
||||||
Source Skill for running or dry-running declared regression and validation gates while capturing durable logs and machine-readable reports.
|
|
||||||
|
|
||||||
The canonical entry point is `SKILL.md`; the deterministic runner is in `scripts/regression_validation_gate_runner.py`.
|
|
||||||
|
|
||||||
## Original Source
|
|
||||||
|
|
||||||
```text
|
|
||||||
Original source: C:\Users\wangq\Documents\Codex\ccpe-system\requirements\skills-vault\2026-06-19-regression-validation-gate-runner.md
|
|
||||||
Migration date: 2026-06-19
|
|
||||||
Migration status: first public Skill implementation
|
|
||||||
Behavior preserved: manifest-declared command execution only; no review judgment or lifecycle approval
|
|
||||||
Known gaps: no project-specific gate presets beyond the example fixture
|
|
||||||
```
|
|
||||||
|
|
||||||
## Layout
|
|
||||||
|
|
||||||
```text
|
|
||||||
SKILL.md
|
|
||||||
README.md
|
|
||||||
agents/openai.yaml
|
|
||||||
fixtures/gate-manifest.example.yaml
|
|
||||||
scripts/regression_validation_gate_runner.py
|
|
||||||
tests/test_regression_validation_gate_runner.py
|
|
||||||
```
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
conda run -n skills-vault python .\scripts\regression_validation_gate_runner.py `
|
|
||||||
--project-root C:\path\project `
|
|
||||||
--gate-manifest C:\path\gate-manifest.yaml `
|
|
||||||
--output-dir C:\path\helper-outputs `
|
|
||||||
--mode dry_run
|
|
||||||
```
|
|
||||||
|
|
||||||
## Tests
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
conda run -n skills-vault python -B -m unittest discover -s skills/regression-validation-gate-runner/tests -v
|
|
||||||
```
|
|
||||||
|
|
@ -1,98 +0,0 @@
|
||||||
---
|
|
||||||
name: regression-validation-gate-runner
|
|
||||||
description: Use when Codex needs to dry-run or execute project-declared regression, validation, test, lint, build, schema, export, or review gates from a JSON, YAML, or Markdown manifest and capture logs before review, release, handoff, CCRA/local review, or lifecycle-sensitive work.
|
|
||||||
---
|
|
||||||
|
|
||||||
# Regression Validation Gate Runner
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
Run or verify explicitly declared validation gates and write a durable evidence package. Use the bundled script instead of ad hoc terminal copying when a reviewer needs command declarations, exit codes, logs, durations, skipped gates, and changed-file notes.
|
|
||||||
|
|
||||||
Passing gates are engineering evidence only. They do not imply product acceptance, lifecycle approval, Owner approval, or CCRA approval.
|
|
||||||
|
|
||||||
## Command
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
conda run -n skills-vault python .\scripts\regression_validation_gate_runner.py `
|
|
||||||
--project-root C:\path\project `
|
|
||||||
--gate-manifest C:\path\gate-manifest.yaml `
|
|
||||||
--output-dir C:\path\helper-outputs `
|
|
||||||
--mode dry_run
|
|
||||||
```
|
|
||||||
|
|
||||||
Use `--mode run` only when the user has asked to execute the manifest commands. Add `--include-optional` only when optional gates should run.
|
|
||||||
|
|
||||||
When installed under `C:\Users\wangq\.agents\skills\regression-validation-gate-runner`, run from that Skill directory or use the installed script path.
|
|
||||||
|
|
||||||
## Manifest
|
|
||||||
|
|
||||||
The manifest may be JSON, YAML, YML, or Markdown containing one fenced `yaml`, `yml`, or `json` block.
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
gates:
|
|
||||||
- gate_id: unit-tests
|
|
||||||
command:
|
|
||||||
- conda
|
|
||||||
- run
|
|
||||||
- -n
|
|
||||||
- skills-vault
|
|
||||||
- python
|
|
||||||
- -B
|
|
||||||
- -m
|
|
||||||
- unittest
|
|
||||||
- discover
|
|
||||||
- -s
|
|
||||||
- skills/example/tests
|
|
||||||
- -v
|
|
||||||
working_directory: .
|
|
||||||
expected_exit_code: 0
|
|
||||||
log_file: logs/unit-tests.log
|
|
||||||
timeout_seconds: 120
|
|
||||||
required_before_review: true
|
|
||||||
```
|
|
||||||
|
|
||||||
`command` may be a string or an argv list. Lists avoid shell quoting surprises on Windows.
|
|
||||||
|
|
||||||
## Modes
|
|
||||||
|
|
||||||
- `dry_run`: parse and validate the manifest, then report declared and skipped gates without executing commands.
|
|
||||||
- `run`: execute required gates. Optional gates are skipped unless `--include-optional` is passed.
|
|
||||||
|
|
||||||
## Outputs
|
|
||||||
|
|
||||||
The script writes under `output_dir`:
|
|
||||||
|
|
||||||
```text
|
|
||||||
gate-run-report.md
|
|
||||||
gate-run-report.json
|
|
||||||
logs/<gate-id>.log
|
|
||||||
```
|
|
||||||
|
|
||||||
Reports include:
|
|
||||||
|
|
||||||
- commands declared, run, and skipped
|
|
||||||
- exit codes and pass/fail/timeout status
|
|
||||||
- duration and log paths
|
|
||||||
- environment notes
|
|
||||||
- project files changed during each gate
|
|
||||||
- machine-readable summary
|
|
||||||
|
|
||||||
Exit code is `0` for `PASS` or `DRY_RUN`, `1` for failed or timed-out required gates, and `2` for manifest/input errors.
|
|
||||||
|
|
||||||
## Safety
|
|
||||||
|
|
||||||
- Run only commands explicitly listed in the manifest.
|
|
||||||
- Keep `working_directory` under `project_root`.
|
|
||||||
- Write logs and reports only under `output_dir`.
|
|
||||||
- Do not install dependencies, invent commands, edit project files directly, or repair failures.
|
|
||||||
- Record changed project files if a command creates, modifies, or deletes files; do not hide side effects.
|
|
||||||
|
|
||||||
## Validation
|
|
||||||
|
|
||||||
After changing runner behavior, run:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
conda run -n skills-vault python -B -m unittest discover -s skills/regression-validation-gate-runner/tests -v
|
|
||||||
conda run -n skills-vault python -B C:\Users\wangq\.codex\skills\.system\skill-creator\scripts\quick_validate.py skills/regression-validation-gate-runner
|
|
||||||
```
|
|
||||||
|
|
@ -1,4 +0,0 @@
|
||||||
interface:
|
|
||||||
display_name: "Regression Validation Gate Runner"
|
|
||||||
short_description: "Run declared validation gates with logs"
|
|
||||||
default_prompt: "Use $regression-validation-gate-runner to dry-run or execute declared project gates and capture review-ready logs."
|
|
||||||
|
|
@ -1,27 +0,0 @@
|
||||||
gates:
|
|
||||||
- gate_id: unit-tests
|
|
||||||
command:
|
|
||||||
- conda
|
|
||||||
- run
|
|
||||||
- -n
|
|
||||||
- skills-vault
|
|
||||||
- python
|
|
||||||
- -B
|
|
||||||
- -m
|
|
||||||
- unittest
|
|
||||||
- discover
|
|
||||||
- -s
|
|
||||||
- skills/example/tests
|
|
||||||
- -v
|
|
||||||
working_directory: .
|
|
||||||
expected_exit_code: 0
|
|
||||||
log_file: logs/unit-tests.log
|
|
||||||
timeout_seconds: 120
|
|
||||||
required_before_review: true
|
|
||||||
- gate_id: packaging-check
|
|
||||||
command: "python -m build --sdist --wheel"
|
|
||||||
working_directory: .
|
|
||||||
expected_exit_code: 0
|
|
||||||
log_file: logs/packaging-check.log
|
|
||||||
timeout_seconds: 300
|
|
||||||
required_before_review: false
|
|
||||||
|
|
@ -1,486 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import pathlib
|
|
||||||
import platform
|
|
||||||
import re
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
from collections.abc import Iterable, Mapping
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import yaml
|
|
||||||
|
|
||||||
|
|
||||||
REPORT_JSON = "gate-run-report.json"
|
|
||||||
REPORT_MD = "gate-run-report.md"
|
|
||||||
|
|
||||||
|
|
||||||
class ManifestError(ValueError):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class Gate:
|
|
||||||
gate_id: str
|
|
||||||
command: str | list[str]
|
|
||||||
working_directory: pathlib.Path
|
|
||||||
expected_exit_code: int
|
|
||||||
log_file: pathlib.Path
|
|
||||||
timeout_seconds: float
|
|
||||||
required_before_review: bool
|
|
||||||
|
|
||||||
|
|
||||||
def load_raw_manifest(path: pathlib.Path) -> Any:
|
|
||||||
text = path.read_text(encoding="utf-8")
|
|
||||||
suffix = path.suffix.lower()
|
|
||||||
if suffix == ".json":
|
|
||||||
return json.loads(text)
|
|
||||||
if suffix in {".yaml", ".yml"}:
|
|
||||||
return yaml.safe_load(text)
|
|
||||||
if suffix in {".md", ".markdown"}:
|
|
||||||
fenced = re.search(r"```(yaml|yml|json)\s*\n(.*?)\n```", text, flags=re.DOTALL | re.I)
|
|
||||||
if not fenced:
|
|
||||||
raise ManifestError("Markdown manifest must contain a fenced yaml, yml, or json block.")
|
|
||||||
language = fenced.group(1).casefold()
|
|
||||||
body = fenced.group(2)
|
|
||||||
return json.loads(body) if language == "json" else yaml.safe_load(body)
|
|
||||||
raise ManifestError("Gate manifest must be JSON, YAML, YML, or Markdown.")
|
|
||||||
|
|
||||||
|
|
||||||
def manifest_gates(raw: Any) -> list[Mapping[str, Any]]:
|
|
||||||
if isinstance(raw, Mapping):
|
|
||||||
gates = raw.get("gates")
|
|
||||||
else:
|
|
||||||
gates = raw
|
|
||||||
if not isinstance(gates, list):
|
|
||||||
raise ManifestError("Gate manifest must define a gates list.")
|
|
||||||
normalized: list[Mapping[str, Any]] = []
|
|
||||||
for index, item in enumerate(gates):
|
|
||||||
if not isinstance(item, Mapping):
|
|
||||||
raise ManifestError(f"Gate at index {index} must be an object.")
|
|
||||||
normalized.append(item)
|
|
||||||
return normalized
|
|
||||||
|
|
||||||
|
|
||||||
def bool_value(value: Any, default: bool) -> bool:
|
|
||||||
if value is None:
|
|
||||||
return default
|
|
||||||
if isinstance(value, bool):
|
|
||||||
return value
|
|
||||||
if isinstance(value, str):
|
|
||||||
return value.strip().casefold() in {"1", "true", "yes", "y"}
|
|
||||||
return bool(value)
|
|
||||||
|
|
||||||
|
|
||||||
def validate_gate_id(value: Any) -> str:
|
|
||||||
if not isinstance(value, str) or not value.strip():
|
|
||||||
raise ManifestError("Every gate must define a non-empty gate_id.")
|
|
||||||
gate_id = value.strip()
|
|
||||||
if any(separator in gate_id for separator in ("/", "\\", os.sep)):
|
|
||||||
raise ManifestError(f"gate_id must not contain path separators: {gate_id}")
|
|
||||||
return gate_id
|
|
||||||
|
|
||||||
|
|
||||||
def validate_command(value: Any, gate_id: str) -> str | list[str]:
|
|
||||||
if isinstance(value, str) and value.strip():
|
|
||||||
return value
|
|
||||||
if isinstance(value, list) and value and all(isinstance(item, str) and item for item in value):
|
|
||||||
return value
|
|
||||||
raise ManifestError(f"Gate {gate_id} must define command as a non-empty string or string list.")
|
|
||||||
|
|
||||||
|
|
||||||
def resolve_under_root(base: pathlib.Path, raw_path: Any, default: str, label: str) -> pathlib.Path:
|
|
||||||
candidate_text = default if raw_path in {None, ""} else str(raw_path)
|
|
||||||
candidate = pathlib.Path(candidate_text)
|
|
||||||
if not candidate.is_absolute():
|
|
||||||
candidate = base / candidate
|
|
||||||
resolved = candidate.resolve()
|
|
||||||
try:
|
|
||||||
resolved.relative_to(base)
|
|
||||||
except ValueError as exc:
|
|
||||||
raise ManifestError(f"{label} must resolve under project_root: {candidate_text}") from exc
|
|
||||||
return resolved
|
|
||||||
|
|
||||||
|
|
||||||
def resolve_log_file(output_dir: pathlib.Path, raw_path: Any, gate_id: str) -> pathlib.Path:
|
|
||||||
candidate_text = f"logs/{gate_id}.log" if raw_path in {None, ""} else str(raw_path)
|
|
||||||
candidate = pathlib.Path(candidate_text)
|
|
||||||
if candidate.is_absolute():
|
|
||||||
resolved = candidate.resolve()
|
|
||||||
else:
|
|
||||||
resolved = (output_dir / candidate).resolve()
|
|
||||||
try:
|
|
||||||
resolved.relative_to(output_dir)
|
|
||||||
except ValueError as exc:
|
|
||||||
raise ManifestError(f"log_file must resolve under output_dir for gate {gate_id}: {candidate_text}") from exc
|
|
||||||
return resolved
|
|
||||||
|
|
||||||
|
|
||||||
def parse_gates(
|
|
||||||
raw_manifest: Any,
|
|
||||||
project_root: pathlib.Path,
|
|
||||||
output_dir: pathlib.Path,
|
|
||||||
) -> tuple[list[Gate], list[str], list[str]]:
|
|
||||||
gates: list[Gate] = []
|
|
||||||
errors: list[str] = []
|
|
||||||
declared: list[str] = []
|
|
||||||
for index, raw_gate in enumerate(manifest_gates(raw_manifest)):
|
|
||||||
try:
|
|
||||||
gate_id = validate_gate_id(raw_gate.get("gate_id"))
|
|
||||||
declared.append(gate_id)
|
|
||||||
command = validate_command(raw_gate.get("command"), gate_id)
|
|
||||||
working_directory = resolve_under_root(
|
|
||||||
project_root,
|
|
||||||
raw_gate.get("working_directory"),
|
|
||||||
".",
|
|
||||||
f"working_directory for gate {gate_id}",
|
|
||||||
)
|
|
||||||
expected_exit_code = int(raw_gate.get("expected_exit_code", 0))
|
|
||||||
timeout_seconds = float(raw_gate.get("timeout_seconds", 300))
|
|
||||||
if timeout_seconds <= 0:
|
|
||||||
raise ManifestError(f"timeout_seconds must be positive for gate {gate_id}.")
|
|
||||||
log_file = resolve_log_file(output_dir, raw_gate.get("log_file"), gate_id)
|
|
||||||
required = bool_value(raw_gate.get("required_before_review"), True)
|
|
||||||
gates.append(
|
|
||||||
Gate(
|
|
||||||
gate_id=gate_id,
|
|
||||||
command=command,
|
|
||||||
working_directory=working_directory,
|
|
||||||
expected_exit_code=expected_exit_code,
|
|
||||||
log_file=log_file,
|
|
||||||
timeout_seconds=timeout_seconds,
|
|
||||||
required_before_review=required,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
except ManifestError as exc:
|
|
||||||
errors.append(f"gate[{index}]: {exc}")
|
|
||||||
return gates, errors, declared
|
|
||||||
|
|
||||||
|
|
||||||
def empty_report(project_root: pathlib.Path, manifest_path: pathlib.Path, mode: str) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"project_root": str(project_root),
|
|
||||||
"gate_manifest_path": str(manifest_path),
|
|
||||||
"mode": mode,
|
|
||||||
"commands_declared": [],
|
|
||||||
"commands_run": [],
|
|
||||||
"commands_skipped": [],
|
|
||||||
"exit_codes": {},
|
|
||||||
"pass_fail_status": "PENDING",
|
|
||||||
"duration": 0.0,
|
|
||||||
"log_paths": {},
|
|
||||||
"environment_notes": [
|
|
||||||
"Commands are declared by the manifest.",
|
|
||||||
"Passing gates are engineering evidence only; they do not imply product acceptance or lifecycle approval.",
|
|
||||||
],
|
|
||||||
"gate_results": [],
|
|
||||||
"manifest_errors": [],
|
|
||||||
"changed_files": [],
|
|
||||||
"machine_readable_summary": {},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def should_exclude(path: pathlib.Path, output_dir: pathlib.Path, project_root: pathlib.Path) -> bool:
|
|
||||||
try:
|
|
||||||
path.relative_to(output_dir)
|
|
||||||
return True
|
|
||||||
except ValueError:
|
|
||||||
pass
|
|
||||||
try:
|
|
||||||
path.relative_to(project_root / ".git")
|
|
||||||
return True
|
|
||||||
except ValueError:
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def snapshot_files(project_root: pathlib.Path, output_dir: pathlib.Path) -> dict[str, tuple[int, int]]:
|
|
||||||
snapshot: dict[str, tuple[int, int]] = {}
|
|
||||||
for path in project_root.rglob("*"):
|
|
||||||
if not path.is_file() or should_exclude(path, output_dir, project_root):
|
|
||||||
continue
|
|
||||||
stat = path.stat()
|
|
||||||
snapshot[path.relative_to(project_root).as_posix()] = (stat.st_size, stat.st_mtime_ns)
|
|
||||||
return snapshot
|
|
||||||
|
|
||||||
|
|
||||||
def diff_snapshots(before: dict[str, tuple[int, int]], after: dict[str, tuple[int, int]]) -> list[str]:
|
|
||||||
changed = {
|
|
||||||
path
|
|
||||||
for path in set(before) | set(after)
|
|
||||||
if before.get(path) != after.get(path)
|
|
||||||
}
|
|
||||||
return sorted(changed)
|
|
||||||
|
|
||||||
|
|
||||||
def command_display(command: str | list[str]) -> str:
|
|
||||||
if isinstance(command, list):
|
|
||||||
return " ".join(command)
|
|
||||||
return command
|
|
||||||
|
|
||||||
|
|
||||||
def command_for_subprocess(command: str | list[str]) -> tuple[str | list[str], bool]:
|
|
||||||
if isinstance(command, list):
|
|
||||||
return command, False
|
|
||||||
return command, True
|
|
||||||
|
|
||||||
|
|
||||||
def safe_text(value: Any) -> str:
|
|
||||||
if value is None:
|
|
||||||
return ""
|
|
||||||
if isinstance(value, bytes):
|
|
||||||
return value.decode("utf-8", errors="replace")
|
|
||||||
return str(value)
|
|
||||||
|
|
||||||
|
|
||||||
def run_gate(
|
|
||||||
gate: Gate,
|
|
||||||
project_root: pathlib.Path,
|
|
||||||
output_dir: pathlib.Path,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
start = time.monotonic()
|
|
||||||
before = snapshot_files(project_root, output_dir)
|
|
||||||
command, shell = command_for_subprocess(gate.command)
|
|
||||||
status = "PENDING"
|
|
||||||
exit_code: int | None = None
|
|
||||||
stdout = ""
|
|
||||||
stderr = ""
|
|
||||||
timeout = False
|
|
||||||
|
|
||||||
try:
|
|
||||||
completed = subprocess.run(
|
|
||||||
command,
|
|
||||||
cwd=str(gate.working_directory),
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
encoding="utf-8",
|
|
||||||
errors="replace",
|
|
||||||
timeout=gate.timeout_seconds,
|
|
||||||
shell=shell,
|
|
||||||
check=False,
|
|
||||||
)
|
|
||||||
exit_code = completed.returncode
|
|
||||||
stdout = completed.stdout
|
|
||||||
stderr = completed.stderr
|
|
||||||
status = "PASS" if exit_code == gate.expected_exit_code else "FAIL"
|
|
||||||
except subprocess.TimeoutExpired as exc:
|
|
||||||
timeout = True
|
|
||||||
stdout = safe_text(exc.stdout or exc.output)
|
|
||||||
stderr = safe_text(exc.stderr)
|
|
||||||
status = "TIMEOUT"
|
|
||||||
duration = time.monotonic() - start
|
|
||||||
after = snapshot_files(project_root, output_dir)
|
|
||||||
changed_files = diff_snapshots(before, after)
|
|
||||||
|
|
||||||
gate.log_file.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
log_lines = [
|
|
||||||
f"gate_id: {gate.gate_id}",
|
|
||||||
f"working_directory: {gate.working_directory}",
|
|
||||||
f"command: {command_display(gate.command)}",
|
|
||||||
f"expected_exit_code: {gate.expected_exit_code}",
|
|
||||||
f"exit_code: {exit_code}",
|
|
||||||
f"status: {status}",
|
|
||||||
f"duration_seconds: {duration:.3f}",
|
|
||||||
"",
|
|
||||||
"## stdout",
|
|
||||||
stdout,
|
|
||||||
"",
|
|
||||||
"## stderr",
|
|
||||||
stderr,
|
|
||||||
]
|
|
||||||
if timeout:
|
|
||||||
log_lines.extend(["", f"Command timed out after {gate.timeout_seconds} seconds."])
|
|
||||||
gate.log_file.write_text("\n".join(log_lines), encoding="utf-8")
|
|
||||||
|
|
||||||
return {
|
|
||||||
"gate_id": gate.gate_id,
|
|
||||||
"command": command_display(gate.command),
|
|
||||||
"working_directory": str(gate.working_directory),
|
|
||||||
"expected_exit_code": gate.expected_exit_code,
|
|
||||||
"exit_code": exit_code,
|
|
||||||
"status": status,
|
|
||||||
"duration": duration,
|
|
||||||
"log_path": gate.log_file.relative_to(output_dir).as_posix(),
|
|
||||||
"required_before_review": gate.required_before_review,
|
|
||||||
"timeout": timeout,
|
|
||||||
"changed_files": changed_files,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def finalize_report(report: dict[str, Any]) -> None:
|
|
||||||
if report["manifest_errors"]:
|
|
||||||
status = "ERROR"
|
|
||||||
elif report["mode"] == "dry_run":
|
|
||||||
status = "DRY_RUN"
|
|
||||||
elif any(result["status"] in {"FAIL", "TIMEOUT"} for result in report["gate_results"]):
|
|
||||||
status = "FAIL"
|
|
||||||
else:
|
|
||||||
status = "PASS"
|
|
||||||
report["pass_fail_status"] = status
|
|
||||||
report["duration"] = round(float(report["duration"]), 3)
|
|
||||||
report["changed_files"] = sorted(
|
|
||||||
{
|
|
||||||
changed
|
|
||||||
for result in report["gate_results"]
|
|
||||||
for changed in result.get("changed_files", [])
|
|
||||||
}
|
|
||||||
)
|
|
||||||
report["machine_readable_summary"] = {
|
|
||||||
"status": status,
|
|
||||||
"commands_declared_count": len(report["commands_declared"]),
|
|
||||||
"commands_run_count": len(report["commands_run"]),
|
|
||||||
"commands_skipped_count": len(report["commands_skipped"]),
|
|
||||||
"manifest_error_count": len(report["manifest_errors"]),
|
|
||||||
"failed_count": sum(1 for result in report["gate_results"] if result["status"] == "FAIL"),
|
|
||||||
"timeout_count": sum(1 for result in report["gate_results"] if result["status"] == "TIMEOUT"),
|
|
||||||
"changed_file_count": len(report["changed_files"]),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def write_json_report(report: dict[str, Any], output_dir: pathlib.Path) -> None:
|
|
||||||
(output_dir / REPORT_JSON).write_text(
|
|
||||||
json.dumps(report, ensure_ascii=False, indent=2),
|
|
||||||
encoding="utf-8",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def write_markdown_report(report: dict[str, Any], output_dir: pathlib.Path) -> None:
|
|
||||||
summary = report["machine_readable_summary"]
|
|
||||||
lines = [
|
|
||||||
"# Gate Run Report",
|
|
||||||
"",
|
|
||||||
"## Summary",
|
|
||||||
"",
|
|
||||||
f"- Status: `{summary['status']}`",
|
|
||||||
f"- Commands declared: {summary['commands_declared_count']}",
|
|
||||||
f"- Commands run: {summary['commands_run_count']}",
|
|
||||||
f"- Commands skipped: {summary['commands_skipped_count']}",
|
|
||||||
f"- Manifest errors: {summary['manifest_error_count']}",
|
|
||||||
f"- Changed files: {summary['changed_file_count']}",
|
|
||||||
"",
|
|
||||||
"## Gate Results",
|
|
||||||
"",
|
|
||||||
]
|
|
||||||
if report["gate_results"]:
|
|
||||||
for result in report["gate_results"]:
|
|
||||||
lines.append(
|
|
||||||
f"- `{result['gate_id']}`: `{result['status']}` exit `{result['exit_code']}` log `{result['log_path']}`"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
lines.append("- None")
|
|
||||||
lines.extend(["", "## Skipped Gates", ""])
|
|
||||||
if report["commands_skipped"]:
|
|
||||||
for skipped in report["commands_skipped"]:
|
|
||||||
lines.append(f"- `{skipped['gate_id']}`: {skipped['reason']}")
|
|
||||||
else:
|
|
||||||
lines.append("- None")
|
|
||||||
lines.extend(["", "## Manifest Errors", ""])
|
|
||||||
if report["manifest_errors"]:
|
|
||||||
for error in report["manifest_errors"]:
|
|
||||||
lines.append(f"- {error}")
|
|
||||||
else:
|
|
||||||
lines.append("- None")
|
|
||||||
lines.extend(["", "## Changed Files", ""])
|
|
||||||
if report["changed_files"]:
|
|
||||||
for path in report["changed_files"]:
|
|
||||||
lines.append(f"- `{path}`")
|
|
||||||
else:
|
|
||||||
lines.append("- None")
|
|
||||||
(output_dir / REPORT_MD).write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
||||||
|
|
||||||
|
|
||||||
def write_reports(report: dict[str, Any], output_dir: pathlib.Path) -> None:
|
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
write_json_report(report, output_dir)
|
|
||||||
write_markdown_report(report, output_dir)
|
|
||||||
|
|
||||||
|
|
||||||
def run(
|
|
||||||
project_root: pathlib.Path,
|
|
||||||
manifest_path: pathlib.Path,
|
|
||||||
output_dir: pathlib.Path,
|
|
||||||
mode: str,
|
|
||||||
include_optional: bool = False,
|
|
||||||
) -> int:
|
|
||||||
project_root = project_root.resolve()
|
|
||||||
manifest_path = manifest_path.resolve()
|
|
||||||
output_dir = output_dir.resolve()
|
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
report = empty_report(project_root, manifest_path, mode)
|
|
||||||
report["environment_notes"].extend(
|
|
||||||
[
|
|
||||||
f"python: {sys.version.split()[0]}",
|
|
||||||
f"platform: {platform.platform()}",
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
if not project_root.is_dir():
|
|
||||||
report["manifest_errors"].append(f"project_root is not a directory: {project_root}")
|
|
||||||
finalize_report(report)
|
|
||||||
write_reports(report, output_dir)
|
|
||||||
return 2
|
|
||||||
if not manifest_path.is_file():
|
|
||||||
report["manifest_errors"].append(f"gate_manifest_path does not exist: {manifest_path}")
|
|
||||||
finalize_report(report)
|
|
||||||
write_reports(report, output_dir)
|
|
||||||
return 2
|
|
||||||
|
|
||||||
try:
|
|
||||||
raw_manifest = load_raw_manifest(manifest_path)
|
|
||||||
gates, errors, declared = parse_gates(raw_manifest, project_root, output_dir)
|
|
||||||
report["commands_declared"] = declared
|
|
||||||
report["manifest_errors"].extend(errors)
|
|
||||||
except Exception as exc:
|
|
||||||
report["manifest_errors"].append(str(exc))
|
|
||||||
finalize_report(report)
|
|
||||||
write_reports(report, output_dir)
|
|
||||||
return 2
|
|
||||||
|
|
||||||
if report["manifest_errors"]:
|
|
||||||
finalize_report(report)
|
|
||||||
write_reports(report, output_dir)
|
|
||||||
return 2
|
|
||||||
|
|
||||||
run_start = time.monotonic()
|
|
||||||
for gate in gates:
|
|
||||||
if mode == "dry_run":
|
|
||||||
report["commands_skipped"].append({"gate_id": gate.gate_id, "reason": "dry_run"})
|
|
||||||
continue
|
|
||||||
if not gate.required_before_review and not include_optional:
|
|
||||||
report["commands_skipped"].append({"gate_id": gate.gate_id, "reason": "optional"})
|
|
||||||
continue
|
|
||||||
result = run_gate(gate, project_root, output_dir)
|
|
||||||
report["gate_results"].append(result)
|
|
||||||
report["commands_run"].append(gate.gate_id)
|
|
||||||
report["exit_codes"][gate.gate_id] = result["exit_code"]
|
|
||||||
report["log_paths"][gate.gate_id] = result["log_path"]
|
|
||||||
|
|
||||||
report["duration"] = time.monotonic() - run_start
|
|
||||||
finalize_report(report)
|
|
||||||
write_reports(report, output_dir)
|
|
||||||
return 0 if report["pass_fail_status"] in {"PASS", "DRY_RUN"} else 1
|
|
||||||
|
|
||||||
|
|
||||||
def main(argv: list[str] | None = None) -> int:
|
|
||||||
parser = argparse.ArgumentParser(description="Run or dry-run declared regression validation gates.")
|
|
||||||
parser.add_argument("--project-root", required=True, type=pathlib.Path)
|
|
||||||
parser.add_argument("--gate-manifest", required=True, type=pathlib.Path)
|
|
||||||
parser.add_argument("--output-dir", required=True, type=pathlib.Path)
|
|
||||||
parser.add_argument("--mode", choices=["dry_run", "run"], required=True)
|
|
||||||
parser.add_argument("--include-optional", action="store_true")
|
|
||||||
args = parser.parse_args(argv)
|
|
||||||
return run(
|
|
||||||
project_root=args.project_root,
|
|
||||||
manifest_path=args.gate_manifest,
|
|
||||||
output_dir=args.output_dir,
|
|
||||||
mode=args.mode,
|
|
||||||
include_optional=args.include_optional,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
|
|
@ -1,306 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
import pathlib
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import tempfile
|
|
||||||
import textwrap
|
|
||||||
import unittest
|
|
||||||
|
|
||||||
|
|
||||||
SKILL_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
||||||
REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]
|
|
||||||
SCRIPT = SKILL_ROOT / "scripts" / "regression_validation_gate_runner.py"
|
|
||||||
TMP_ROOT = REPO_ROOT / "tmp" / "regression-validation-gate-runner-tests"
|
|
||||||
|
|
||||||
|
|
||||||
def quote(value: str) -> str:
|
|
||||||
return json.dumps(value)
|
|
||||||
|
|
||||||
|
|
||||||
def py_command(source: str) -> list[str]:
|
|
||||||
return [sys.executable, "-c", source]
|
|
||||||
|
|
||||||
|
|
||||||
def run_cli(*args: str) -> subprocess.CompletedProcess[str]:
|
|
||||||
return subprocess.run(
|
|
||||||
[sys.executable, str(SCRIPT), *args],
|
|
||||||
text=True,
|
|
||||||
capture_output=True,
|
|
||||||
check=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def write_text(path: pathlib.Path, text: str) -> pathlib.Path:
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
path.write_text(text, encoding="utf-8")
|
|
||||||
return path
|
|
||||||
|
|
||||||
|
|
||||||
def write_json(path: pathlib.Path, data: object) -> pathlib.Path:
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
||||||
return path
|
|
||||||
|
|
||||||
|
|
||||||
def load_report(output_dir: pathlib.Path) -> dict[str, object]:
|
|
||||||
return json.loads((output_dir / "gate-run-report.json").read_text(encoding="utf-8"))
|
|
||||||
|
|
||||||
|
|
||||||
class RegressionValidationGateRunnerTest(unittest.TestCase):
|
|
||||||
def setUp(self) -> None:
|
|
||||||
TMP_ROOT.mkdir(parents=True, exist_ok=True)
|
|
||||||
self.tmp_dir = tempfile.TemporaryDirectory(dir=TMP_ROOT)
|
|
||||||
self.root = pathlib.Path(self.tmp_dir.name) / "project"
|
|
||||||
self.output_dir = pathlib.Path(self.tmp_dir.name) / "outputs"
|
|
||||||
self.manifest = pathlib.Path(self.tmp_dir.name) / "gates.json"
|
|
||||||
self.root.mkdir()
|
|
||||||
|
|
||||||
def tearDown(self) -> None:
|
|
||||||
self.tmp_dir.cleanup()
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def tearDownClass(cls) -> None:
|
|
||||||
if TMP_ROOT.exists():
|
|
||||||
shutil.rmtree(TMP_ROOT)
|
|
||||||
|
|
||||||
def run_gate_runner(self, mode: str = "run") -> tuple[subprocess.CompletedProcess[str], dict[str, object]]:
|
|
||||||
result = run_cli(
|
|
||||||
"--project-root",
|
|
||||||
str(self.root),
|
|
||||||
"--gate-manifest",
|
|
||||||
str(self.manifest),
|
|
||||||
"--output-dir",
|
|
||||||
str(self.output_dir),
|
|
||||||
"--mode",
|
|
||||||
mode,
|
|
||||||
)
|
|
||||||
return result, load_report(self.output_dir)
|
|
||||||
|
|
||||||
def test_dry_run_parses_json_manifest_without_running_commands(self) -> None:
|
|
||||||
marker = self.root / "should-not-exist.txt"
|
|
||||||
write_json(
|
|
||||||
self.manifest,
|
|
||||||
{
|
|
||||||
"gates": [
|
|
||||||
{
|
|
||||||
"gate_id": "unit",
|
|
||||||
"command": py_command(f"open({quote(str(marker))}, 'w').write('ran')"),
|
|
||||||
"working_directory": ".",
|
|
||||||
"expected_exit_code": 0,
|
|
||||||
"timeout_seconds": 5,
|
|
||||||
"required_before_review": True,
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
result, report = self.run_gate_runner(mode="dry_run")
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertFalse(marker.exists())
|
|
||||||
self.assertEqual(report["commands_declared"], ["unit"])
|
|
||||||
self.assertEqual(report["commands_run"], [])
|
|
||||||
self.assertEqual(report["commands_skipped"][0]["reason"], "dry_run")
|
|
||||||
self.assertEqual(report["machine_readable_summary"]["status"], "DRY_RUN")
|
|
||||||
|
|
||||||
def test_successful_command_capture_writes_log_and_passes(self) -> None:
|
|
||||||
write_json(
|
|
||||||
self.manifest,
|
|
||||||
{
|
|
||||||
"gates": [
|
|
||||||
{
|
|
||||||
"gate_id": "unit",
|
|
||||||
"command": py_command("print('hello gate')"),
|
|
||||||
"working_directory": ".",
|
|
||||||
"expected_exit_code": 0,
|
|
||||||
"log_file": "logs/unit.log",
|
|
||||||
"timeout_seconds": 5,
|
|
||||||
"required_before_review": True,
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
result, report = self.run_gate_runner()
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual(report["pass_fail_status"], "PASS")
|
|
||||||
self.assertEqual(report["exit_codes"]["unit"], 0)
|
|
||||||
log_path = self.output_dir / "logs" / "unit.log"
|
|
||||||
self.assertTrue(log_path.exists())
|
|
||||||
self.assertIn("hello gate", log_path.read_text(encoding="utf-8"))
|
|
||||||
self.assertEqual(report["commands_run"], ["unit"])
|
|
||||||
|
|
||||||
def test_failing_command_capture_fails_required_gate(self) -> None:
|
|
||||||
write_json(
|
|
||||||
self.manifest,
|
|
||||||
{
|
|
||||||
"gates": [
|
|
||||||
{
|
|
||||||
"gate_id": "regression",
|
|
||||||
"command": py_command("import sys; print('bad'); sys.exit(3)"),
|
|
||||||
"expected_exit_code": 0,
|
|
||||||
"timeout_seconds": 5,
|
|
||||||
"required_before_review": True,
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
result, report = self.run_gate_runner()
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 1, result.stderr)
|
|
||||||
self.assertEqual(report["pass_fail_status"], "FAIL")
|
|
||||||
self.assertEqual(report["exit_codes"]["regression"], 3)
|
|
||||||
gate = report["gate_results"][0]
|
|
||||||
self.assertEqual(gate["status"], "FAIL")
|
|
||||||
self.assertIn("bad", (self.output_dir / "logs" / "regression.log").read_text(encoding="utf-8"))
|
|
||||||
|
|
||||||
def test_optional_gate_is_skipped_by_default(self) -> None:
|
|
||||||
marker = self.root / "optional-ran.txt"
|
|
||||||
write_json(
|
|
||||||
self.manifest,
|
|
||||||
{
|
|
||||||
"gates": [
|
|
||||||
{
|
|
||||||
"gate_id": "optional-packaging",
|
|
||||||
"command": py_command(f"open({quote(str(marker))}, 'w').write('ran')"),
|
|
||||||
"expected_exit_code": 0,
|
|
||||||
"timeout_seconds": 5,
|
|
||||||
"required_before_review": False,
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
result, report = self.run_gate_runner()
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertFalse(marker.exists())
|
|
||||||
self.assertEqual(report["commands_run"], [])
|
|
||||||
self.assertEqual(report["commands_skipped"][0]["gate_id"], "optional-packaging")
|
|
||||||
self.assertEqual(report["commands_skipped"][0]["reason"], "optional")
|
|
||||||
|
|
||||||
def test_missing_command_field_is_manifest_error(self) -> None:
|
|
||||||
write_json(
|
|
||||||
self.manifest,
|
|
||||||
{"gates": [{"gate_id": "broken", "expected_exit_code": 0, "required_before_review": True}]},
|
|
||||||
)
|
|
||||||
|
|
||||||
result, report = self.run_gate_runner()
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 2)
|
|
||||||
self.assertEqual(report["pass_fail_status"], "ERROR")
|
|
||||||
self.assertEqual(report["machine_readable_summary"]["manifest_error_count"], 1)
|
|
||||||
self.assertIn("command", report["manifest_errors"][0])
|
|
||||||
|
|
||||||
def test_timeout_handling_marks_required_gate_failed(self) -> None:
|
|
||||||
write_json(
|
|
||||||
self.manifest,
|
|
||||||
{
|
|
||||||
"gates": [
|
|
||||||
{
|
|
||||||
"gate_id": "slow",
|
|
||||||
"command": py_command("import time; time.sleep(3)"),
|
|
||||||
"expected_exit_code": 0,
|
|
||||||
"timeout_seconds": 1,
|
|
||||||
"required_before_review": True,
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
result, report = self.run_gate_runner()
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 1, result.stderr)
|
|
||||||
gate = report["gate_results"][0]
|
|
||||||
self.assertEqual(gate["status"], "TIMEOUT")
|
|
||||||
self.assertEqual(report["pass_fail_status"], "FAIL")
|
|
||||||
self.assertIn("timed out", (self.output_dir / "logs" / "slow.log").read_text(encoding="utf-8"))
|
|
||||||
|
|
||||||
def test_markdown_manifest_with_yaml_fence_is_supported(self) -> None:
|
|
||||||
self.manifest = pathlib.Path(self.tmp_dir.name) / "gates.md"
|
|
||||||
write_text(
|
|
||||||
self.manifest,
|
|
||||||
textwrap.dedent(
|
|
||||||
f"""\
|
|
||||||
# Gates
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
gates:
|
|
||||||
- gate_id: markdown-gate
|
|
||||||
command:
|
|
||||||
- {sys.executable}
|
|
||||||
- -c
|
|
||||||
- "print('from markdown')"
|
|
||||||
expected_exit_code: 0
|
|
||||||
timeout_seconds: 5
|
|
||||||
required_before_review: true
|
|
||||||
```
|
|
||||||
"""
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
result, report = self.run_gate_runner()
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual(report["commands_declared"], ["markdown-gate"])
|
|
||||||
self.assertEqual(report["commands_run"], ["markdown-gate"])
|
|
||||||
|
|
||||||
def test_utf8_paths_and_chinese_filenames_are_logged(self) -> None:
|
|
||||||
project = pathlib.Path(self.tmp_dir.name) / "项目"
|
|
||||||
project.mkdir()
|
|
||||||
self.root = project
|
|
||||||
write_json(
|
|
||||||
self.manifest,
|
|
||||||
{
|
|
||||||
"gates": [
|
|
||||||
{
|
|
||||||
"gate_id": "中文-gate",
|
|
||||||
"command": py_command("print('中文输出')"),
|
|
||||||
"working_directory": ".",
|
|
||||||
"expected_exit_code": 0,
|
|
||||||
"timeout_seconds": 5,
|
|
||||||
"required_before_review": True,
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
result, report = self.run_gate_runner()
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
log_path = self.output_dir / "logs" / "中文-gate.log"
|
|
||||||
self.assertTrue(log_path.exists())
|
|
||||||
self.assertIn("中文输出", log_path.read_text(encoding="utf-8"))
|
|
||||||
self.assertIn("中文-gate", report["log_paths"])
|
|
||||||
|
|
||||||
def test_project_file_changes_are_reported(self) -> None:
|
|
||||||
changed = self.root / "generated.txt"
|
|
||||||
write_json(
|
|
||||||
self.manifest,
|
|
||||||
{
|
|
||||||
"gates": [
|
|
||||||
{
|
|
||||||
"gate_id": "side-effect",
|
|
||||||
"command": py_command(f"open({quote(str(changed))}, 'w', encoding='utf-8').write('new')"),
|
|
||||||
"expected_exit_code": 0,
|
|
||||||
"timeout_seconds": 5,
|
|
||||||
"required_before_review": True,
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
result, report = self.run_gate_runner()
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual(report["changed_files"], ["generated.txt"])
|
|
||||||
self.assertEqual(report["gate_results"][0]["changed_files"], ["generated.txt"])
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
unittest.main()
|
|
||||||
|
|
@ -1,41 +0,0 @@
|
||||||
# Review Bundle Audit
|
|
||||||
|
|
||||||
Source Skill for auditing review bundle directories before human or Agent review.
|
|
||||||
|
|
||||||
The canonical entry point is `SKILL.md`; the deterministic auditor is in `scripts/review_bundle_audit.py`.
|
|
||||||
|
|
||||||
## Original Source
|
|
||||||
|
|
||||||
```text
|
|
||||||
Original source: C:\Users\wangq\Documents\Codex\ccpe-system\requirements\skills-vault\2026-06-19-review-bundle-audit.md
|
|
||||||
Migration date: 2026-06-19
|
|
||||||
Migration status: first public Skill implementation
|
|
||||||
Behavior preserved: deterministic package audit only; no review judgment or bundle repair
|
|
||||||
Known gaps: built-in profile is generic and intentionally conservative
|
|
||||||
```
|
|
||||||
|
|
||||||
## Layout
|
|
||||||
|
|
||||||
```text
|
|
||||||
SKILL.md
|
|
||||||
README.md
|
|
||||||
agents/openai.yaml
|
|
||||||
fixtures/profile.example.yaml
|
|
||||||
scripts/review_bundle_audit.py
|
|
||||||
tests/test_review_bundle_audit.py
|
|
||||||
```
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
conda run -n skills-vault python .\scripts\review_bundle_audit.py `
|
|
||||||
--bundle-root C:\path\review-bundle `
|
|
||||||
--output-dir C:\path\helper-outputs `
|
|
||||||
--profile generic
|
|
||||||
```
|
|
||||||
|
|
||||||
## Tests
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
conda run -n skills-vault python -B -m unittest discover -s skills/review-bundle-audit/tests -v
|
|
||||||
```
|
|
||||||
|
|
@ -1,103 +0,0 @@
|
||||||
---
|
|
||||||
name: review-bundle-audit
|
|
||||||
description: Use when Codex needs to preflight a review, handoff, release, CCRA, Web upload, article, video, Skill, or code-review bundle directory for required files, manifests, validation sidecars, reports, zip readability, missing materials, extra files, UTF-8 paths, and review-package completeness before a human or Agent review.
|
|
||||||
---
|
|
||||||
|
|
||||||
# Review Bundle Audit
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
Audit a review bundle directory before reviewer judgment is spent. Use the bundled script to produce deterministic Markdown and JSON reports about bundle structure, required files, readable manifests, validation sidecars, reports, zips, warnings, and blockers.
|
|
||||||
|
|
||||||
This Skill is a package audit only. It must not repair bundles, edit source files, or decide review acceptance.
|
|
||||||
|
|
||||||
## Command
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
conda run -n skills-vault python .\scripts\review_bundle_audit.py `
|
|
||||||
--bundle-root C:\path\review-bundle `
|
|
||||||
--output-dir C:\path\helper-outputs `
|
|
||||||
--profile generic
|
|
||||||
```
|
|
||||||
|
|
||||||
Use `--profile-config C:\path\profile.yaml` when a project has its own required file names.
|
|
||||||
|
|
||||||
When installed under `C:\Users\wangq\.agents\skills\review-bundle-audit`, run from that Skill directory or use the installed script path.
|
|
||||||
|
|
||||||
## Built-In Profile
|
|
||||||
|
|
||||||
`generic` requires:
|
|
||||||
|
|
||||||
- `brief`: `*brief*.md`, `*brief*.markdown`
|
|
||||||
- `manifest`: `*manifest*.json`, `*manifest*.yaml`, `*manifest*.yml`
|
|
||||||
- `validation_sidecar`: `*validation*sidecar*.json`, `*validation*sidecar*.yaml`, `*validation*sidecar*.yml`
|
|
||||||
|
|
||||||
It optionally recognizes:
|
|
||||||
|
|
||||||
- `report`: `*report*.md`, `*report*.markdown`
|
|
||||||
- `zip`: `*.zip`
|
|
||||||
- `changed_files`: changed-file summary names
|
|
||||||
|
|
||||||
Extra files produce warnings, not blockers. Missing report or zip produces warnings. Unreadable manifest, sidecar, or zip produces a blocker.
|
|
||||||
|
|
||||||
## Profile Config
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
profiles:
|
|
||||||
project-review:
|
|
||||||
required_files:
|
|
||||||
brief:
|
|
||||||
- review-brief.md
|
|
||||||
manifest:
|
|
||||||
- review-manifest.json
|
|
||||||
validation_sidecar:
|
|
||||||
- validation-sidecar.json
|
|
||||||
optional_files:
|
|
||||||
report:
|
|
||||||
- "*report*.md"
|
|
||||||
zip:
|
|
||||||
- "*.zip"
|
|
||||||
allowed_extra_patterns:
|
|
||||||
- notes/*.md
|
|
||||||
```
|
|
||||||
|
|
||||||
## Outputs
|
|
||||||
|
|
||||||
The script writes under `output_dir`:
|
|
||||||
|
|
||||||
```text
|
|
||||||
bundle-audit.md
|
|
||||||
bundle-audit.json
|
|
||||||
```
|
|
||||||
|
|
||||||
Reports include:
|
|
||||||
|
|
||||||
- bundle root and profile
|
|
||||||
- files discovered
|
|
||||||
- required files present and missing
|
|
||||||
- optional files present
|
|
||||||
- manifest status
|
|
||||||
- zip status
|
|
||||||
- validation sidecar status
|
|
||||||
- report status
|
|
||||||
- warnings and blocking errors
|
|
||||||
- machine-readable summary
|
|
||||||
|
|
||||||
Exit code is `0` when no blockers exist, `1` for audit blockers, and `2` for input/profile errors.
|
|
||||||
|
|
||||||
## Safety
|
|
||||||
|
|
||||||
- Read only the specified bundle directory and optional profile config.
|
|
||||||
- Write only audit reports under `output_dir`.
|
|
||||||
- Do not create, repair, normalize, or delete bundle files.
|
|
||||||
- Do not edit model cards, selector rules, regression cases, project docs, or source files.
|
|
||||||
- Treat the audit as evidence for a reviewer, not as review acceptance.
|
|
||||||
|
|
||||||
## Validation
|
|
||||||
|
|
||||||
After changing audit behavior, run:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
conda run -n skills-vault python -B -m unittest discover -s skills/review-bundle-audit/tests -v
|
|
||||||
conda run -n skills-vault python -B C:\Users\wangq\.codex\skills\.system\skill-creator\scripts\quick_validate.py skills/review-bundle-audit
|
|
||||||
```
|
|
||||||
|
|
@ -1,4 +0,0 @@
|
||||||
interface:
|
|
||||||
display_name: "Review Bundle Audit"
|
|
||||||
short_description: "Preflight review bundle completeness"
|
|
||||||
default_prompt: "Use $review-bundle-audit to preflight a review bundle and write Markdown and JSON audit reports."
|
|
||||||
|
|
@ -1,23 +0,0 @@
|
||||||
profiles:
|
|
||||||
project-review:
|
|
||||||
required_files:
|
|
||||||
brief:
|
|
||||||
- review-brief.md
|
|
||||||
- "*brief*.md"
|
|
||||||
manifest:
|
|
||||||
- review-manifest.json
|
|
||||||
- "*manifest*.json"
|
|
||||||
validation_sidecar:
|
|
||||||
- validation-sidecar.json
|
|
||||||
- "*validation*sidecar*.json"
|
|
||||||
optional_files:
|
|
||||||
report:
|
|
||||||
- "*report*.md"
|
|
||||||
zip:
|
|
||||||
- "*.zip"
|
|
||||||
changed_files:
|
|
||||||
- "*changed*files*.md"
|
|
||||||
- "*changed-files*.json"
|
|
||||||
allowed_extra_patterns:
|
|
||||||
- notes/*.md
|
|
||||||
- README.md
|
|
||||||
|
|
@ -1,367 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import fnmatch
|
|
||||||
import json
|
|
||||||
import pathlib
|
|
||||||
import sys
|
|
||||||
import zipfile
|
|
||||||
from collections.abc import Iterable, Mapping
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import yaml
|
|
||||||
|
|
||||||
|
|
||||||
REPORT_JSON = "bundle-audit.json"
|
|
||||||
REPORT_MD = "bundle-audit.md"
|
|
||||||
|
|
||||||
|
|
||||||
BUILTIN_PROFILES: dict[str, dict[str, Any]] = {
|
|
||||||
"generic": {
|
|
||||||
"required_files": {
|
|
||||||
"brief": ["*brief*.md", "*brief*.markdown"],
|
|
||||||
"manifest": ["*manifest*.json", "*manifest*.yaml", "*manifest*.yml"],
|
|
||||||
"validation_sidecar": [
|
|
||||||
"*validation*sidecar*.json",
|
|
||||||
"*validation*sidecar*.yaml",
|
|
||||||
"*validation*sidecar*.yml",
|
|
||||||
],
|
|
||||||
},
|
|
||||||
"optional_files": {
|
|
||||||
"report": ["*report*.md", "*report*.markdown"],
|
|
||||||
"zip": ["*.zip"],
|
|
||||||
"changed_files": ["*changed*files*.md", "*changed-files*.json", "*changed*files*.json"],
|
|
||||||
},
|
|
||||||
"allowed_extra_patterns": [],
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def as_list(value: Any) -> list[str]:
|
|
||||||
if value is None:
|
|
||||||
return []
|
|
||||||
if isinstance(value, str):
|
|
||||||
return [value]
|
|
||||||
if isinstance(value, Iterable):
|
|
||||||
return [str(item) for item in value if item is not None]
|
|
||||||
return [str(value)]
|
|
||||||
|
|
||||||
|
|
||||||
def load_structured(path: pathlib.Path) -> Any:
|
|
||||||
text = path.read_text(encoding="utf-8")
|
|
||||||
suffix = path.suffix.casefold()
|
|
||||||
if suffix == ".json":
|
|
||||||
return json.loads(text)
|
|
||||||
if suffix in {".yaml", ".yml"}:
|
|
||||||
return yaml.safe_load(text)
|
|
||||||
return text
|
|
||||||
|
|
||||||
|
|
||||||
def load_profile_config(path: pathlib.Path | None) -> dict[str, Any]:
|
|
||||||
if path is None:
|
|
||||||
return {}
|
|
||||||
parsed = load_structured(path)
|
|
||||||
if not isinstance(parsed, Mapping):
|
|
||||||
raise ValueError("Profile config must be a JSON or YAML object.")
|
|
||||||
return dict(parsed)
|
|
||||||
|
|
||||||
|
|
||||||
def get_profile(profile_name: str, config: Mapping[str, Any]) -> dict[str, Any]:
|
|
||||||
profiles = dict(BUILTIN_PROFILES)
|
|
||||||
configured_profiles = config.get("profiles") if isinstance(config, Mapping) else None
|
|
||||||
if isinstance(configured_profiles, Mapping):
|
|
||||||
for name, profile in configured_profiles.items():
|
|
||||||
if isinstance(profile, Mapping):
|
|
||||||
profiles[str(name)] = dict(profile)
|
|
||||||
if profile_name not in profiles:
|
|
||||||
raise ValueError(f"Unknown profile: {profile_name}")
|
|
||||||
return normalize_profile(profiles[profile_name])
|
|
||||||
|
|
||||||
|
|
||||||
def normalize_pattern_map(raw: Any) -> dict[str, list[str]]:
|
|
||||||
if not isinstance(raw, Mapping):
|
|
||||||
return {}
|
|
||||||
return {str(key): as_list(value) for key, value in raw.items()}
|
|
||||||
|
|
||||||
|
|
||||||
def normalize_profile(raw_profile: Mapping[str, Any]) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"required_files": normalize_pattern_map(raw_profile.get("required_files")),
|
|
||||||
"optional_files": normalize_pattern_map(raw_profile.get("optional_files")),
|
|
||||||
"allowed_extra_patterns": as_list(raw_profile.get("allowed_extra_patterns")),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def rel_path(path: pathlib.Path, root: pathlib.Path) -> str:
|
|
||||||
return path.relative_to(root).as_posix()
|
|
||||||
|
|
||||||
|
|
||||||
def discover_files(bundle_root: pathlib.Path) -> list[pathlib.Path]:
|
|
||||||
return sorted(path for path in bundle_root.rglob("*") if path.is_file())
|
|
||||||
|
|
||||||
|
|
||||||
def match_patterns(file_rel: str, patterns: list[str]) -> bool:
|
|
||||||
basename = pathlib.PurePosixPath(file_rel).name
|
|
||||||
return any(fnmatch.fnmatch(file_rel, pattern) or fnmatch.fnmatch(basename, pattern) for pattern in patterns)
|
|
||||||
|
|
||||||
|
|
||||||
def files_for_patterns(files_rel: list[str], patterns: list[str]) -> list[str]:
|
|
||||||
return sorted(file_rel for file_rel in files_rel if match_patterns(file_rel, patterns))
|
|
||||||
|
|
||||||
|
|
||||||
def status_for_category(files: list[str], missing_status: str = "missing") -> dict[str, Any]:
|
|
||||||
return {"status": "present" if files else missing_status, "files": files}
|
|
||||||
|
|
||||||
|
|
||||||
def inspect_structured_files(
|
|
||||||
bundle_root: pathlib.Path,
|
|
||||||
file_rels: list[str],
|
|
||||||
missing_status: str = "missing",
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
if not file_rels:
|
|
||||||
return {"status": missing_status, "files": []}
|
|
||||||
errors: list[str] = []
|
|
||||||
for file_rel in file_rels:
|
|
||||||
try:
|
|
||||||
load_structured(bundle_root / file_rel)
|
|
||||||
except Exception as exc:
|
|
||||||
errors.append(f"{file_rel}: {exc}")
|
|
||||||
if errors:
|
|
||||||
return {"status": "unreadable", "files": file_rels, "errors": errors}
|
|
||||||
return {"status": "present", "files": file_rels}
|
|
||||||
|
|
||||||
|
|
||||||
def inspect_zip_files(bundle_root: pathlib.Path, zip_rels: list[str]) -> dict[str, Any]:
|
|
||||||
if not zip_rels:
|
|
||||||
return {"status": "missing", "files": []}
|
|
||||||
errors: list[str] = []
|
|
||||||
entry_counts: dict[str, int] = {}
|
|
||||||
for file_rel in zip_rels:
|
|
||||||
path = bundle_root / file_rel
|
|
||||||
try:
|
|
||||||
with zipfile.ZipFile(path) as archive:
|
|
||||||
bad_member = archive.testzip()
|
|
||||||
if bad_member is not None:
|
|
||||||
errors.append(f"{file_rel}: unreadable member {bad_member}")
|
|
||||||
entry_counts[file_rel] = len(archive.namelist())
|
|
||||||
except Exception as exc:
|
|
||||||
errors.append(f"{file_rel}: {exc}")
|
|
||||||
if errors:
|
|
||||||
return {"status": "unreadable", "files": zip_rels, "errors": errors, "entry_counts": entry_counts}
|
|
||||||
return {"status": "readable", "files": zip_rels, "entry_counts": entry_counts}
|
|
||||||
|
|
||||||
|
|
||||||
def category_matches(profile: Mapping[str, Any], files_rel: list[str]) -> tuple[dict[str, list[str]], dict[str, list[str]]]:
|
|
||||||
required_matches = {
|
|
||||||
category: files_for_patterns(files_rel, patterns)
|
|
||||||
for category, patterns in profile["required_files"].items()
|
|
||||||
}
|
|
||||||
optional_matches = {
|
|
||||||
category: files_for_patterns(files_rel, patterns)
|
|
||||||
for category, patterns in profile["optional_files"].items()
|
|
||||||
}
|
|
||||||
return required_matches, optional_matches
|
|
||||||
|
|
||||||
|
|
||||||
def allowed_patterns(profile: Mapping[str, Any]) -> list[str]:
|
|
||||||
patterns: list[str] = []
|
|
||||||
for pattern_map_name in ("required_files", "optional_files"):
|
|
||||||
for category_patterns in profile[pattern_map_name].values():
|
|
||||||
patterns.extend(category_patterns)
|
|
||||||
patterns.extend(profile.get("allowed_extra_patterns", []))
|
|
||||||
return patterns
|
|
||||||
|
|
||||||
|
|
||||||
def extra_files(files_rel: list[str], profile: Mapping[str, Any]) -> list[str]:
|
|
||||||
patterns = allowed_patterns(profile)
|
|
||||||
return [file_rel for file_rel in files_rel if not match_patterns(file_rel, patterns)]
|
|
||||||
|
|
||||||
|
|
||||||
def empty_report(bundle_root: pathlib.Path, profile_name: str) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"bundle_root": str(bundle_root),
|
|
||||||
"profile": profile_name,
|
|
||||||
"files_discovered": [],
|
|
||||||
"required_files_present": {},
|
|
||||||
"required_files_missing": [],
|
|
||||||
"optional_files_present": {},
|
|
||||||
"manifest_status": {"status": "missing", "files": []},
|
|
||||||
"zip_status": {"status": "missing", "files": []},
|
|
||||||
"validation_sidecar_status": {"status": "missing", "files": []},
|
|
||||||
"report_status": {"status": "missing", "files": []},
|
|
||||||
"warnings": [],
|
|
||||||
"blocking_errors": [],
|
|
||||||
"machine_readable_summary": {},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def audit_bundle(bundle_root: pathlib.Path, profile_name: str, profile: Mapping[str, Any]) -> dict[str, Any]:
|
|
||||||
report = empty_report(bundle_root, profile_name)
|
|
||||||
files = discover_files(bundle_root)
|
|
||||||
files_rel = [rel_path(path, bundle_root) for path in files]
|
|
||||||
report["files_discovered"] = files_rel
|
|
||||||
|
|
||||||
required_matches, optional_matches = category_matches(profile, files_rel)
|
|
||||||
report["required_files_present"] = {
|
|
||||||
category: matches for category, matches in required_matches.items() if matches
|
|
||||||
}
|
|
||||||
report["optional_files_present"] = {
|
|
||||||
category: matches for category, matches in optional_matches.items() if matches
|
|
||||||
}
|
|
||||||
|
|
||||||
for category, matches in required_matches.items():
|
|
||||||
if not matches:
|
|
||||||
report["required_files_missing"].append(category)
|
|
||||||
report["blocking_errors"].append(f"Missing required {category} file.")
|
|
||||||
|
|
||||||
manifest_files = required_matches.get("manifest", [])
|
|
||||||
sidecar_files = required_matches.get("validation_sidecar", [])
|
|
||||||
report_files = optional_matches.get("report", [])
|
|
||||||
zip_files = optional_matches.get("zip", [])
|
|
||||||
|
|
||||||
report["manifest_status"] = inspect_structured_files(bundle_root, manifest_files)
|
|
||||||
report["validation_sidecar_status"] = inspect_structured_files(bundle_root, sidecar_files)
|
|
||||||
report["report_status"] = status_for_category(report_files)
|
|
||||||
report["zip_status"] = inspect_zip_files(bundle_root, zip_files)
|
|
||||||
|
|
||||||
if report["manifest_status"]["status"] == "unreadable":
|
|
||||||
report["blocking_errors"].append("Manifest file is unreadable.")
|
|
||||||
if report["validation_sidecar_status"]["status"] == "unreadable":
|
|
||||||
report["blocking_errors"].append("Validation sidecar file is unreadable.")
|
|
||||||
if report["zip_status"]["status"] == "unreadable":
|
|
||||||
report["blocking_errors"].append("Zip file is unreadable.")
|
|
||||||
if not zip_files:
|
|
||||||
report["warnings"].append("No zip file matched the profile.")
|
|
||||||
if not report_files:
|
|
||||||
report["warnings"].append("No review report file matched the profile.")
|
|
||||||
|
|
||||||
for file_rel in extra_files(files_rel, profile):
|
|
||||||
report["warnings"].append(f"Extra file not matched by profile: {file_rel}")
|
|
||||||
|
|
||||||
report["required_files_missing"] = sorted(report["required_files_missing"])
|
|
||||||
report["warnings"] = sorted(report["warnings"])
|
|
||||||
report["blocking_errors"] = sorted(set(report["blocking_errors"]))
|
|
||||||
finalize_report(report)
|
|
||||||
return report
|
|
||||||
|
|
||||||
|
|
||||||
def finalize_report(report: dict[str, Any]) -> None:
|
|
||||||
status = "FAIL" if report["blocking_errors"] else "PASS"
|
|
||||||
report["machine_readable_summary"] = {
|
|
||||||
"status": status,
|
|
||||||
"files_discovered_count": len(report["files_discovered"]),
|
|
||||||
"required_missing_count": len(report["required_files_missing"]),
|
|
||||||
"warning_count": len(report["warnings"]),
|
|
||||||
"blocking_error_count": len(report["blocking_errors"]),
|
|
||||||
"manifest_status": report["manifest_status"]["status"],
|
|
||||||
"zip_status": report["zip_status"]["status"],
|
|
||||||
"validation_sidecar_status": report["validation_sidecar_status"]["status"],
|
|
||||||
"report_status": report["report_status"]["status"],
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def write_json_report(report: Mapping[str, Any], output_dir: pathlib.Path) -> None:
|
|
||||||
(output_dir / REPORT_JSON).write_text(
|
|
||||||
json.dumps(report, ensure_ascii=False, indent=2),
|
|
||||||
encoding="utf-8",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def write_markdown_report(report: Mapping[str, Any], output_dir: pathlib.Path) -> None:
|
|
||||||
summary = report["machine_readable_summary"]
|
|
||||||
lines = [
|
|
||||||
"# Review Bundle Audit",
|
|
||||||
"",
|
|
||||||
"## Summary",
|
|
||||||
"",
|
|
||||||
f"- Status: `{summary['status']}`",
|
|
||||||
f"- Bundle root: `{report['bundle_root']}`",
|
|
||||||
f"- Profile: `{report['profile']}`",
|
|
||||||
f"- Files discovered: {summary['files_discovered_count']}",
|
|
||||||
f"- Required files missing: {summary['required_missing_count']}",
|
|
||||||
f"- Warnings: {summary['warning_count']}",
|
|
||||||
f"- Blocking errors: {summary['blocking_error_count']}",
|
|
||||||
"",
|
|
||||||
"## Required Files",
|
|
||||||
"",
|
|
||||||
]
|
|
||||||
for category, files in report["required_files_present"].items():
|
|
||||||
lines.append(f"- `{category}`: {', '.join(f'`{file}`' for file in files)}")
|
|
||||||
if report["required_files_missing"]:
|
|
||||||
lines.append("")
|
|
||||||
lines.append("Missing:")
|
|
||||||
for category in report["required_files_missing"]:
|
|
||||||
lines.append(f"- `{category}`")
|
|
||||||
lines.extend(["", "## Package Status", ""])
|
|
||||||
for key in ("manifest_status", "validation_sidecar_status", "zip_status", "report_status"):
|
|
||||||
status = report[key]["status"]
|
|
||||||
files = report[key].get("files", [])
|
|
||||||
rendered_files = ", ".join(f"`{file}`" for file in files) if files else "none"
|
|
||||||
lines.append(f"- `{key}`: `{status}` ({rendered_files})")
|
|
||||||
lines.extend(["", "## Blocking Errors", ""])
|
|
||||||
if report["blocking_errors"]:
|
|
||||||
for error in report["blocking_errors"]:
|
|
||||||
lines.append(f"- {error}")
|
|
||||||
else:
|
|
||||||
lines.append("- None")
|
|
||||||
lines.extend(["", "## Warnings", ""])
|
|
||||||
if report["warnings"]:
|
|
||||||
for warning in report["warnings"]:
|
|
||||||
lines.append(f"- {warning}")
|
|
||||||
else:
|
|
||||||
lines.append("- None")
|
|
||||||
lines.extend(["", "## Files Discovered", ""])
|
|
||||||
for file_rel in report["files_discovered"]:
|
|
||||||
lines.append(f"- `{file_rel}`")
|
|
||||||
(output_dir / REPORT_MD).write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
||||||
|
|
||||||
|
|
||||||
def write_reports(report: Mapping[str, Any], output_dir: pathlib.Path) -> None:
|
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
write_json_report(report, output_dir)
|
|
||||||
write_markdown_report(report, output_dir)
|
|
||||||
|
|
||||||
|
|
||||||
def run(
|
|
||||||
bundle_root: pathlib.Path,
|
|
||||||
output_dir: pathlib.Path,
|
|
||||||
profile_name: str,
|
|
||||||
profile_config: pathlib.Path | None,
|
|
||||||
) -> int:
|
|
||||||
bundle_root = bundle_root.resolve()
|
|
||||||
output_dir = output_dir.resolve()
|
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
report = empty_report(bundle_root, profile_name)
|
|
||||||
|
|
||||||
if not bundle_root.is_dir():
|
|
||||||
report["blocking_errors"].append(f"Bundle root is not a directory: {bundle_root}")
|
|
||||||
finalize_report(report)
|
|
||||||
write_reports(report, output_dir)
|
|
||||||
return 2
|
|
||||||
|
|
||||||
try:
|
|
||||||
config = load_profile_config(profile_config)
|
|
||||||
profile = get_profile(profile_name, config)
|
|
||||||
report = audit_bundle(bundle_root, profile_name, profile)
|
|
||||||
except Exception as exc:
|
|
||||||
report["blocking_errors"].append(str(exc))
|
|
||||||
finalize_report(report)
|
|
||||||
write_reports(report, output_dir)
|
|
||||||
return 2
|
|
||||||
|
|
||||||
write_reports(report, output_dir)
|
|
||||||
return 1 if report["blocking_errors"] else 0
|
|
||||||
|
|
||||||
|
|
||||||
def main(argv: list[str] | None = None) -> int:
|
|
||||||
parser = argparse.ArgumentParser(description="Audit a review bundle directory.")
|
|
||||||
parser.add_argument("--bundle-root", required=True, type=pathlib.Path)
|
|
||||||
parser.add_argument("--output-dir", required=True, type=pathlib.Path)
|
|
||||||
parser.add_argument("--profile", default="generic")
|
|
||||||
parser.add_argument("--profile-config", type=pathlib.Path)
|
|
||||||
args = parser.parse_args(argv)
|
|
||||||
return run(args.bundle_root, args.output_dir, args.profile, args.profile_config)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
|
|
@ -1,196 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
import pathlib
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import tempfile
|
|
||||||
import unittest
|
|
||||||
import zipfile
|
|
||||||
|
|
||||||
|
|
||||||
SKILL_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
||||||
REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]
|
|
||||||
SCRIPT = SKILL_ROOT / "scripts" / "review_bundle_audit.py"
|
|
||||||
TMP_ROOT = REPO_ROOT / "tmp" / "review-bundle-audit-tests"
|
|
||||||
|
|
||||||
|
|
||||||
def run_cli(*args: str) -> subprocess.CompletedProcess[str]:
|
|
||||||
return subprocess.run(
|
|
||||||
[sys.executable, str(SCRIPT), *args],
|
|
||||||
text=True,
|
|
||||||
capture_output=True,
|
|
||||||
check=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def write_text(path: pathlib.Path, text: str = "content") -> pathlib.Path:
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
path.write_text(text, encoding="utf-8")
|
|
||||||
return path
|
|
||||||
|
|
||||||
|
|
||||||
def write_json(path: pathlib.Path, data: object) -> pathlib.Path:
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
||||||
return path
|
|
||||||
|
|
||||||
|
|
||||||
def write_zip(path: pathlib.Path) -> pathlib.Path:
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
with zipfile.ZipFile(path, "w") as archive:
|
|
||||||
archive.writestr("included.txt", "ok")
|
|
||||||
return path
|
|
||||||
|
|
||||||
|
|
||||||
def load_report(output_dir: pathlib.Path) -> dict[str, object]:
|
|
||||||
return json.loads((output_dir / "bundle-audit.json").read_text(encoding="utf-8"))
|
|
||||||
|
|
||||||
|
|
||||||
class ReviewBundleAuditTest(unittest.TestCase):
|
|
||||||
def setUp(self) -> None:
|
|
||||||
TMP_ROOT.mkdir(parents=True, exist_ok=True)
|
|
||||||
self.tmp_dir = tempfile.TemporaryDirectory(dir=TMP_ROOT)
|
|
||||||
self.bundle_root = pathlib.Path(self.tmp_dir.name) / "ccra_review_bundle" / "round-05"
|
|
||||||
self.output_dir = pathlib.Path(self.tmp_dir.name) / "outputs"
|
|
||||||
self.bundle_root.mkdir(parents=True)
|
|
||||||
|
|
||||||
def tearDown(self) -> None:
|
|
||||||
self.tmp_dir.cleanup()
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def tearDownClass(cls) -> None:
|
|
||||||
if TMP_ROOT.exists():
|
|
||||||
shutil.rmtree(TMP_ROOT)
|
|
||||||
|
|
||||||
def write_complete_bundle(self) -> None:
|
|
||||||
write_text(self.bundle_root / "review-brief.md", "# Brief\n")
|
|
||||||
write_json(self.bundle_root / "review-manifest.json", {"files": ["review-brief.md"]})
|
|
||||||
write_json(self.bundle_root / "validation-sidecar.json", {"status": "pass"})
|
|
||||||
write_text(self.bundle_root / "local-review-report.md", "# Report\n")
|
|
||||||
write_text(self.bundle_root / "changed-files.md", "- src/model.json\n")
|
|
||||||
write_zip(self.bundle_root / "raw-review-bundle.zip")
|
|
||||||
|
|
||||||
def run_audit(self, *extra_args: str) -> tuple[subprocess.CompletedProcess[str], dict[str, object]]:
|
|
||||||
result = run_cli(
|
|
||||||
"--bundle-root",
|
|
||||||
str(self.bundle_root),
|
|
||||||
"--output-dir",
|
|
||||||
str(self.output_dir),
|
|
||||||
*extra_args,
|
|
||||||
)
|
|
||||||
return result, load_report(self.output_dir)
|
|
||||||
|
|
||||||
def test_complete_bundle_passes_and_reports_present_files(self) -> None:
|
|
||||||
self.write_complete_bundle()
|
|
||||||
|
|
||||||
result, report = self.run_audit()
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual(report["machine_readable_summary"]["status"], "PASS")
|
|
||||||
self.assertEqual(report["required_files_missing"], [])
|
|
||||||
self.assertEqual(report["manifest_status"]["status"], "present")
|
|
||||||
self.assertEqual(report["zip_status"]["status"], "readable")
|
|
||||||
self.assertEqual(report["validation_sidecar_status"]["status"], "present")
|
|
||||||
self.assertEqual(report["report_status"]["status"], "present")
|
|
||||||
self.assertIn("review-brief.md", report["files_discovered"])
|
|
||||||
|
|
||||||
def test_missing_required_brief_is_blocking(self) -> None:
|
|
||||||
self.write_complete_bundle()
|
|
||||||
(self.bundle_root / "review-brief.md").unlink()
|
|
||||||
|
|
||||||
result, report = self.run_audit()
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 1, result.stderr)
|
|
||||||
self.assertEqual(report["machine_readable_summary"]["status"], "FAIL")
|
|
||||||
self.assertIn("brief", report["required_files_missing"])
|
|
||||||
self.assertTrue(any("brief" in error for error in report["blocking_errors"]))
|
|
||||||
|
|
||||||
def test_missing_manifest_is_blocking(self) -> None:
|
|
||||||
self.write_complete_bundle()
|
|
||||||
(self.bundle_root / "review-manifest.json").unlink()
|
|
||||||
|
|
||||||
result, report = self.run_audit()
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 1, result.stderr)
|
|
||||||
self.assertEqual(report["manifest_status"]["status"], "missing")
|
|
||||||
self.assertIn("manifest", report["required_files_missing"])
|
|
||||||
|
|
||||||
def test_missing_validation_sidecar_is_blocking(self) -> None:
|
|
||||||
self.write_complete_bundle()
|
|
||||||
(self.bundle_root / "validation-sidecar.json").unlink()
|
|
||||||
|
|
||||||
result, report = self.run_audit()
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 1, result.stderr)
|
|
||||||
self.assertEqual(report["validation_sidecar_status"]["status"], "missing")
|
|
||||||
self.assertIn("validation_sidecar", report["required_files_missing"])
|
|
||||||
|
|
||||||
def test_unreadable_zip_is_blocking_when_zip_is_present(self) -> None:
|
|
||||||
self.write_complete_bundle()
|
|
||||||
write_text(self.bundle_root / "raw-review-bundle.zip", "not a real zip")
|
|
||||||
|
|
||||||
result, report = self.run_audit()
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 1, result.stderr)
|
|
||||||
self.assertEqual(report["zip_status"]["status"], "unreadable")
|
|
||||||
self.assertTrue(any("zip" in error.casefold() for error in report["blocking_errors"]))
|
|
||||||
|
|
||||||
def test_extra_files_are_warnings_not_blockers(self) -> None:
|
|
||||||
self.write_complete_bundle()
|
|
||||||
write_text(self.bundle_root / "scratch.tmp", "operator note")
|
|
||||||
|
|
||||||
result, report = self.run_audit()
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual(report["machine_readable_summary"]["status"], "PASS")
|
|
||||||
self.assertEqual(report["blocking_errors"], [])
|
|
||||||
self.assertTrue(any("scratch.tmp" in warning for warning in report["warnings"]))
|
|
||||||
|
|
||||||
def test_utf8_paths_and_chinese_filenames_are_reported(self) -> None:
|
|
||||||
self.bundle_root = pathlib.Path(self.tmp_dir.name) / "审查包"
|
|
||||||
self.bundle_root.mkdir()
|
|
||||||
write_text(self.bundle_root / "审查-brief.md", "# Brief\n")
|
|
||||||
write_json(self.bundle_root / "审查-manifest.json", {"files": ["审查-brief.md"]})
|
|
||||||
write_json(self.bundle_root / "验证-validation-sidecar.json", {"status": "pass"})
|
|
||||||
write_text(self.bundle_root / "审查-report.md", "# Report\n")
|
|
||||||
write_zip(self.bundle_root / "原始-review-bundle.zip")
|
|
||||||
|
|
||||||
result, report = self.run_audit()
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertIn("审查-brief.md", report["files_discovered"])
|
|
||||||
self.assertEqual(report["zip_status"]["files"][0], "原始-review-bundle.zip")
|
|
||||||
|
|
||||||
def test_profile_config_can_define_custom_required_patterns(self) -> None:
|
|
||||||
profile_config = pathlib.Path(self.tmp_dir.name) / "profile.json"
|
|
||||||
write_text(self.bundle_root / "handoff.txt", "handoff")
|
|
||||||
write_json(self.bundle_root / "index.json", {"files": ["handoff.txt"]})
|
|
||||||
write_json(self.bundle_root / "checks.json", {"status": "pass"})
|
|
||||||
write_json(
|
|
||||||
profile_config,
|
|
||||||
{
|
|
||||||
"profiles": {
|
|
||||||
"custom": {
|
|
||||||
"required_files": {
|
|
||||||
"brief": ["handoff.txt"],
|
|
||||||
"manifest": ["index.json"],
|
|
||||||
"validation_sidecar": ["checks.json"],
|
|
||||||
},
|
|
||||||
"optional_files": {"report": ["notes.md"], "zip": ["*.zip"]},
|
|
||||||
"allowed_extra_patterns": ["handoff.txt", "index.json", "checks.json"],
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
result, report = self.run_audit("--profile", "custom", "--profile-config", str(profile_config))
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual(report["profile"], "custom")
|
|
||||||
self.assertEqual(report["required_files_missing"], [])
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
unittest.main()
|
|
||||||
|
|
@ -1,43 +0,0 @@
|
||||||
# Review Context Builder
|
|
||||||
|
|
||||||
Source Skill for generating file-first review context indexes and manifests.
|
|
||||||
|
|
||||||
The canonical entry point is `SKILL.md`; the deterministic builder is in `scripts/review_context_builder.py`.
|
|
||||||
|
|
||||||
## Original Source
|
|
||||||
|
|
||||||
```text
|
|
||||||
Original source: C:\Users\wangq\Documents\Codex\ccpe-system\requirements\skills-vault\2026-06-19-review-context-builder.md
|
|
||||||
Migration date: 2026-06-19
|
|
||||||
Migration status: first public Skill implementation
|
|
||||||
Behavior preserved: deterministic context indexing only; no review judgment or source evidence replacement
|
|
||||||
Known gaps: categorization uses filename/path heuristics only
|
|
||||||
```
|
|
||||||
|
|
||||||
## Layout
|
|
||||||
|
|
||||||
```text
|
|
||||||
SKILL.md
|
|
||||||
README.md
|
|
||||||
agents/openai.yaml
|
|
||||||
fixtures/metadata.example.json
|
|
||||||
scripts/review_context_builder.py
|
|
||||||
tests/test_review_context_builder.py
|
|
||||||
```
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
conda run -n skills-vault python .\scripts\review_context_builder.py `
|
|
||||||
--project-root C:\path\project `
|
|
||||||
--review-id review-001 `
|
|
||||||
--source-root . `
|
|
||||||
--include-pattern "**/*.md" `
|
|
||||||
--output-dir C:\path\helper-outputs
|
|
||||||
```
|
|
||||||
|
|
||||||
## Tests
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
conda run -n skills-vault python -B -m unittest discover -s skills/review-context-builder/tests -v
|
|
||||||
```
|
|
||||||
|
|
@ -1,87 +0,0 @@
|
||||||
---
|
|
||||||
name: review-context-builder
|
|
||||||
description: Use when Codex needs to build a file-first review, audit, planning, synthesis, release, handoff, CCRA, article, video, Skill, model-card, PR, or agent-invocation context package from configured local files, reports, diffs, validation outputs, prior decisions, metadata, include/exclude patterns, and source roots.
|
|
||||||
---
|
|
||||||
|
|
||||||
# Review Context Builder
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
Build a deterministic context index and file manifest before reviewer or Agent invocation. Use the bundled script to identify the files a reviewer should read, categorize references, pass metadata through, and record missing or excluded sources.
|
|
||||||
|
|
||||||
This Skill creates navigation context only. It must not summarize source evidence in a way that replaces file reading, make findings, create bundles, or decide readiness.
|
|
||||||
|
|
||||||
## Command
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
conda run -n skills-vault python .\scripts\review_context_builder.py `
|
|
||||||
--project-root C:\path\project `
|
|
||||||
--review-id round-05-local-pass `
|
|
||||||
--source-root README.md `
|
|
||||||
--source-root reports `
|
|
||||||
--include-pattern "**/*.md" `
|
|
||||||
--exclude-pattern "archive/**" `
|
|
||||||
--metadata C:\path\metadata.json `
|
|
||||||
--output-dir C:\path\helper-outputs
|
|
||||||
```
|
|
||||||
|
|
||||||
When installed under `C:\Users\wangq\.agents\skills\review-context-builder`, run from that Skill directory or use the installed script path.
|
|
||||||
|
|
||||||
## Inputs
|
|
||||||
|
|
||||||
- `--project-root`: root used for path normalization and source-root containment.
|
|
||||||
- `--review-id`: stable ID for the review or handoff context.
|
|
||||||
- `--context-profile`: optional label, default `generic`.
|
|
||||||
- `--source-root`: repeatable file or directory, relative to `project_root` or absolute under it.
|
|
||||||
- `--include-pattern`: repeatable glob, default `**/*`.
|
|
||||||
- `--exclude-pattern`: repeatable glob for files to record as excluded.
|
|
||||||
- `--metadata`: optional JSON/YAML object; passed through without hard-coded CCRA fields.
|
|
||||||
- `--max-file-bytes`: files larger than this are excluded and recorded.
|
|
||||||
|
|
||||||
## Outputs
|
|
||||||
|
|
||||||
The script writes under `output_dir`:
|
|
||||||
|
|
||||||
```text
|
|
||||||
review-context.md
|
|
||||||
review-file-manifest.json
|
|
||||||
```
|
|
||||||
|
|
||||||
Outputs include:
|
|
||||||
|
|
||||||
- review ID and context profile
|
|
||||||
- source roots and missing source roots
|
|
||||||
- files included and excluded
|
|
||||||
- important report references
|
|
||||||
- test or validation output references
|
|
||||||
- diff or changed-file references
|
|
||||||
- prior decision references
|
|
||||||
- known non-goals and open questions from metadata
|
|
||||||
- original metadata passthrough
|
|
||||||
- machine-readable summary
|
|
||||||
|
|
||||||
## Rules
|
|
||||||
|
|
||||||
- Normalize included paths as project-relative POSIX paths.
|
|
||||||
- Keep output ordering stable.
|
|
||||||
- Treat missing source roots as blockers while still writing both output files.
|
|
||||||
- Exclude large files instead of embedding content.
|
|
||||||
- Do not read or quote source file bodies for context prose.
|
|
||||||
- Do not inject project-specific fields unless they came from metadata.
|
|
||||||
|
|
||||||
## Safety
|
|
||||||
|
|
||||||
- Read only configured source roots and optional metadata file.
|
|
||||||
- Write only `review-context.md` and `review-file-manifest.json` under `output_dir`.
|
|
||||||
- Do not edit project files.
|
|
||||||
- Do not create review bundles.
|
|
||||||
- Do not decide review acceptance, round closure, lifecycle status, release readiness, or task approval.
|
|
||||||
|
|
||||||
## Validation
|
|
||||||
|
|
||||||
After changing builder behavior, run:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
conda run -n skills-vault python -B -m unittest discover -s skills/review-context-builder/tests -v
|
|
||||||
conda run -n skills-vault python -B C:\Users\wangq\.codex\skills\.system\skill-creator\scripts\quick_validate.py skills/review-context-builder
|
|
||||||
```
|
|
||||||
|
|
@ -1,4 +0,0 @@
|
||||||
interface:
|
|
||||||
display_name: "Review Context Builder"
|
|
||||||
short_description: "Build file-first review context indexes"
|
|
||||||
default_prompt: "Use $review-context-builder to create a review-context.md and review-file-manifest.json from configured local evidence files."
|
|
||||||
|
|
@ -1,12 +0,0 @@
|
||||||
{
|
|
||||||
"review_goal": "Check whether the supplied evidence is ready for reviewer judgment.",
|
|
||||||
"non_goals": [
|
|
||||||
"Do not decide approval.",
|
|
||||||
"Do not replace reading source files."
|
|
||||||
],
|
|
||||||
"open_questions": [
|
|
||||||
"Are required evidence files present?",
|
|
||||||
"Are validation outputs current?"
|
|
||||||
],
|
|
||||||
"target_reviewer": "reviewer-or-agent-name"
|
|
||||||
}
|
|
||||||
|
|
@ -1,371 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import fnmatch
|
|
||||||
import json
|
|
||||||
import pathlib
|
|
||||||
from collections.abc import Mapping
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import yaml
|
|
||||||
|
|
||||||
|
|
||||||
CONTEXT_MD = "review-context.md"
|
|
||||||
MANIFEST_JSON = "review-file-manifest.json"
|
|
||||||
DEFAULT_MAX_FILE_BYTES = 1_000_000
|
|
||||||
|
|
||||||
|
|
||||||
def as_list(value: Any) -> list[str]:
|
|
||||||
if value is None:
|
|
||||||
return []
|
|
||||||
if isinstance(value, list):
|
|
||||||
return [str(item) for item in value]
|
|
||||||
return [str(value)]
|
|
||||||
|
|
||||||
|
|
||||||
def load_metadata(path: pathlib.Path | None) -> dict[str, Any]:
|
|
||||||
if path is None:
|
|
||||||
return {}
|
|
||||||
text = path.read_text(encoding="utf-8")
|
|
||||||
if path.suffix.casefold() == ".json":
|
|
||||||
parsed = json.loads(text)
|
|
||||||
else:
|
|
||||||
parsed = yaml.safe_load(text)
|
|
||||||
if parsed is None:
|
|
||||||
return {}
|
|
||||||
if not isinstance(parsed, Mapping):
|
|
||||||
raise ValueError("Metadata must be a JSON or YAML object.")
|
|
||||||
return dict(parsed)
|
|
||||||
|
|
||||||
|
|
||||||
def rel_path(path: pathlib.Path, project_root: pathlib.Path) -> str:
|
|
||||||
return path.relative_to(project_root).as_posix()
|
|
||||||
|
|
||||||
|
|
||||||
def resolve_source_root(raw_source: str, project_root: pathlib.Path) -> pathlib.Path:
|
|
||||||
source = pathlib.Path(raw_source)
|
|
||||||
if not source.is_absolute():
|
|
||||||
source = project_root / source
|
|
||||||
return source.resolve()
|
|
||||||
|
|
||||||
|
|
||||||
def normalize_source_root(raw_source: str, project_root: pathlib.Path) -> str:
|
|
||||||
source = resolve_source_root(raw_source, project_root)
|
|
||||||
try:
|
|
||||||
return rel_path(source, project_root)
|
|
||||||
except ValueError:
|
|
||||||
return str(source)
|
|
||||||
|
|
||||||
|
|
||||||
def path_matches(path: str, patterns: list[str]) -> bool:
|
|
||||||
if not patterns:
|
|
||||||
return True
|
|
||||||
basename = pathlib.PurePosixPath(path).name
|
|
||||||
for pattern in patterns:
|
|
||||||
root_pattern = pattern[3:] if pattern.startswith("**/") else None
|
|
||||||
if fnmatch.fnmatch(path, pattern) or fnmatch.fnmatch(basename, pattern):
|
|
||||||
return True
|
|
||||||
if root_pattern and (fnmatch.fnmatch(path, root_pattern) or fnmatch.fnmatch(basename, root_pattern)):
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def iter_source_files(source: pathlib.Path) -> list[pathlib.Path]:
|
|
||||||
if source.is_file():
|
|
||||||
return [source]
|
|
||||||
if source.is_dir():
|
|
||||||
return sorted(path for path in source.rglob("*") if path.is_file())
|
|
||||||
return []
|
|
||||||
|
|
||||||
|
|
||||||
def empty_manifest(
|
|
||||||
project_root: pathlib.Path,
|
|
||||||
review_id: str,
|
|
||||||
context_profile: str,
|
|
||||||
source_roots: list[str],
|
|
||||||
metadata: dict[str, Any],
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"review_id": review_id,
|
|
||||||
"context_profile": context_profile,
|
|
||||||
"project_root": str(project_root),
|
|
||||||
"source_roots": source_roots,
|
|
||||||
"missing_source_roots": [],
|
|
||||||
"files_included": [],
|
|
||||||
"files_excluded": [],
|
|
||||||
"important_reports": [],
|
|
||||||
"test_or_validation_outputs": [],
|
|
||||||
"diff_or_changed_file_references": [],
|
|
||||||
"prior_decision_references": [],
|
|
||||||
"known_non_goals": as_list(metadata.get("known_non_goals", metadata.get("non_goals"))),
|
|
||||||
"open_questions_for_reviewer_or_agent": as_list(
|
|
||||||
metadata.get("open_questions_for_reviewer_or_agent", metadata.get("open_questions"))
|
|
||||||
),
|
|
||||||
"metadata": metadata,
|
|
||||||
"blocking_errors": [],
|
|
||||||
"machine_readable_summary": {},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def classify_paths(included_paths: list[str]) -> dict[str, list[str]]:
|
|
||||||
important_reports: list[str] = []
|
|
||||||
validation_outputs: list[str] = []
|
|
||||||
diffs: list[str] = []
|
|
||||||
decisions: list[str] = []
|
|
||||||
|
|
||||||
for path in included_paths:
|
|
||||||
folded = path.casefold()
|
|
||||||
name = pathlib.PurePosixPath(path).name.casefold()
|
|
||||||
if any(token in folded for token in ("report", "audit", "review")):
|
|
||||||
important_reports.append(path)
|
|
||||||
if any(token in folded for token in ("test", "validation", "gate", "log")):
|
|
||||||
validation_outputs.append(path)
|
|
||||||
if any(token in folded for token in ("diff", "changed-file", "changed_files", "changed-files")):
|
|
||||||
diffs.append(path)
|
|
||||||
if any(token in folded for token in ("decision", "owner-decision", "approval")) or "decision" in name:
|
|
||||||
decisions.append(path)
|
|
||||||
|
|
||||||
return {
|
|
||||||
"important_reports": sorted(set(important_reports)),
|
|
||||||
"test_or_validation_outputs": sorted(set(validation_outputs)),
|
|
||||||
"diff_or_changed_file_references": sorted(set(diffs)),
|
|
||||||
"prior_decision_references": sorted(set(decisions)),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def build_manifest(
|
|
||||||
project_root: pathlib.Path,
|
|
||||||
review_id: str,
|
|
||||||
context_profile: str,
|
|
||||||
source_roots_raw: list[str],
|
|
||||||
include_patterns: list[str],
|
|
||||||
exclude_patterns: list[str],
|
|
||||||
metadata: dict[str, Any],
|
|
||||||
max_file_bytes: int,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
normalized_source_roots = [
|
|
||||||
normalize_source_root(raw_source, project_root) for raw_source in source_roots_raw
|
|
||||||
]
|
|
||||||
manifest = empty_manifest(project_root, review_id, context_profile, normalized_source_roots, metadata)
|
|
||||||
seen: dict[str, pathlib.Path] = {}
|
|
||||||
|
|
||||||
for raw_source in source_roots_raw:
|
|
||||||
source = resolve_source_root(raw_source, project_root)
|
|
||||||
normalized = normalize_source_root(raw_source, project_root)
|
|
||||||
if not source.exists():
|
|
||||||
manifest["missing_source_roots"].append(normalized)
|
|
||||||
manifest["blocking_errors"].append(f"Missing source root: {normalized}")
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
source.relative_to(project_root)
|
|
||||||
except ValueError:
|
|
||||||
manifest["missing_source_roots"].append(normalized)
|
|
||||||
manifest["blocking_errors"].append(f"Source root is outside project_root: {normalized}")
|
|
||||||
continue
|
|
||||||
for file_path in iter_source_files(source):
|
|
||||||
rel = rel_path(file_path.resolve(), project_root)
|
|
||||||
seen[rel] = file_path
|
|
||||||
|
|
||||||
included: list[dict[str, Any]] = []
|
|
||||||
excluded: list[dict[str, Any]] = []
|
|
||||||
for file_rel in sorted(seen):
|
|
||||||
file_path = seen[file_rel]
|
|
||||||
if not path_matches(file_rel, include_patterns):
|
|
||||||
continue
|
|
||||||
if exclude_patterns and path_matches(file_rel, exclude_patterns):
|
|
||||||
excluded.append({"path": file_rel, "reason": "exclude_pattern"})
|
|
||||||
continue
|
|
||||||
size = file_size_bytes(file_path)
|
|
||||||
if size > max_file_bytes:
|
|
||||||
excluded.append({"path": file_rel, "reason": "size_exceeds_limit", "size_bytes": size})
|
|
||||||
continue
|
|
||||||
source_root = source_root_for_file(file_path, source_roots_raw, project_root)
|
|
||||||
included.append({"path": file_rel, "size_bytes": size, "source_root": source_root})
|
|
||||||
|
|
||||||
manifest["files_included"] = included
|
|
||||||
manifest["files_excluded"] = sorted(excluded, key=lambda item: item["path"])
|
|
||||||
categories = classify_paths([item["path"] for item in included])
|
|
||||||
manifest.update(categories)
|
|
||||||
finalize_manifest(manifest)
|
|
||||||
return manifest
|
|
||||||
|
|
||||||
|
|
||||||
def source_root_for_file(file_path: pathlib.Path, source_roots_raw: list[str], project_root: pathlib.Path) -> str:
|
|
||||||
file_path = file_path.resolve()
|
|
||||||
best: tuple[int, str] | None = None
|
|
||||||
for raw_source in source_roots_raw:
|
|
||||||
source = resolve_source_root(raw_source, project_root)
|
|
||||||
if source.is_file() and source == file_path:
|
|
||||||
root_rel = normalize_source_root(raw_source, project_root)
|
|
||||||
score = len(pathlib.PurePosixPath(root_rel).parts)
|
|
||||||
if best is None or score > best[0]:
|
|
||||||
best = (score, root_rel)
|
|
||||||
elif source.is_dir():
|
|
||||||
try:
|
|
||||||
file_path.relative_to(source)
|
|
||||||
except ValueError:
|
|
||||||
continue
|
|
||||||
root_rel = normalize_source_root(raw_source, project_root)
|
|
||||||
score = len(pathlib.PurePosixPath(root_rel).parts)
|
|
||||||
if best is None or score > best[0]:
|
|
||||||
best = (score, root_rel)
|
|
||||||
return "." if best is None or best[1] == "" else best[1]
|
|
||||||
|
|
||||||
|
|
||||||
def file_size_bytes(path: pathlib.Path) -> int:
|
|
||||||
return len(path.read_text(encoding="utf-8", errors="replace").encode("utf-8"))
|
|
||||||
|
|
||||||
|
|
||||||
def finalize_manifest(manifest: dict[str, Any]) -> None:
|
|
||||||
status = "FAIL" if manifest["blocking_errors"] else "PASS"
|
|
||||||
manifest["machine_readable_summary"] = {
|
|
||||||
"status": status,
|
|
||||||
"source_roots_count": len(manifest["source_roots"]),
|
|
||||||
"missing_source_roots_count": len(manifest["missing_source_roots"]),
|
|
||||||
"files_included_count": len(manifest["files_included"]),
|
|
||||||
"files_excluded_count": len(manifest["files_excluded"]),
|
|
||||||
"important_reports_count": len(manifest["important_reports"]),
|
|
||||||
"validation_outputs_count": len(manifest["test_or_validation_outputs"]),
|
|
||||||
"diff_references_count": len(manifest["diff_or_changed_file_references"]),
|
|
||||||
"prior_decisions_count": len(manifest["prior_decision_references"]),
|
|
||||||
"blocking_error_count": len(manifest["blocking_errors"]),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def write_manifest(manifest: dict[str, Any], output_dir: pathlib.Path) -> None:
|
|
||||||
(output_dir / MANIFEST_JSON).write_text(
|
|
||||||
json.dumps(manifest, ensure_ascii=False, indent=2),
|
|
||||||
encoding="utf-8",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def write_context(manifest: dict[str, Any], output_dir: pathlib.Path) -> None:
|
|
||||||
lines = [
|
|
||||||
"# Review Context",
|
|
||||||
"",
|
|
||||||
"This context is an index. It does not replace reading source files.",
|
|
||||||
"",
|
|
||||||
"## Metadata",
|
|
||||||
"",
|
|
||||||
f"- Review ID: `{manifest['review_id']}`",
|
|
||||||
f"- Context profile: `{manifest['context_profile']}`",
|
|
||||||
f"- Status: `{manifest['machine_readable_summary']['status']}`",
|
|
||||||
"",
|
|
||||||
"## Source Roots",
|
|
||||||
"",
|
|
||||||
]
|
|
||||||
for source in manifest["source_roots"]:
|
|
||||||
lines.append(f"- `{source}`")
|
|
||||||
if manifest["missing_source_roots"]:
|
|
||||||
lines.extend(["", "## Missing Source Roots", ""])
|
|
||||||
for source in manifest["missing_source_roots"]:
|
|
||||||
lines.append(f"- `{source}`")
|
|
||||||
lines.extend(["", "## Files Included", ""])
|
|
||||||
if manifest["files_included"]:
|
|
||||||
for item in manifest["files_included"]:
|
|
||||||
lines.append(f"- `{item['path']}` ({item['size_bytes']} bytes)")
|
|
||||||
else:
|
|
||||||
lines.append("- None")
|
|
||||||
lines.extend(["", "## Files Excluded", ""])
|
|
||||||
if manifest["files_excluded"]:
|
|
||||||
for item in manifest["files_excluded"]:
|
|
||||||
detail = f", {item['size_bytes']} bytes" if "size_bytes" in item else ""
|
|
||||||
lines.append(f"- `{item['path']}`: {item['reason']}{detail}")
|
|
||||||
else:
|
|
||||||
lines.append("- None")
|
|
||||||
for title, key in [
|
|
||||||
("Important Reports", "important_reports"),
|
|
||||||
("Test Or Validation Outputs", "test_or_validation_outputs"),
|
|
||||||
("Diff Or Changed File References", "diff_or_changed_file_references"),
|
|
||||||
("Prior Decision References", "prior_decision_references"),
|
|
||||||
("Known Non Goals", "known_non_goals"),
|
|
||||||
("Open Questions For Reviewer Or Agent", "open_questions_for_reviewer_or_agent"),
|
|
||||||
]:
|
|
||||||
lines.extend(["", f"## {title}", ""])
|
|
||||||
values = manifest[key]
|
|
||||||
if values:
|
|
||||||
for value in values:
|
|
||||||
lines.append(f"- `{value}`" if isinstance(value, str) else f"- {value}")
|
|
||||||
else:
|
|
||||||
lines.append("- None")
|
|
||||||
lines.extend(["", "## Blocking Errors", ""])
|
|
||||||
if manifest["blocking_errors"]:
|
|
||||||
for error in manifest["blocking_errors"]:
|
|
||||||
lines.append(f"- {error}")
|
|
||||||
else:
|
|
||||||
lines.append("- None")
|
|
||||||
lines.extend(["", "## Machine Readable Manifest", "", f"- `{MANIFEST_JSON}`"])
|
|
||||||
(output_dir / CONTEXT_MD).write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
||||||
|
|
||||||
|
|
||||||
def write_outputs(manifest: dict[str, Any], output_dir: pathlib.Path) -> None:
|
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
write_manifest(manifest, output_dir)
|
|
||||||
write_context(manifest, output_dir)
|
|
||||||
|
|
||||||
|
|
||||||
def run(
|
|
||||||
project_root: pathlib.Path,
|
|
||||||
review_id: str,
|
|
||||||
output_dir: pathlib.Path,
|
|
||||||
source_roots: list[str],
|
|
||||||
include_patterns: list[str],
|
|
||||||
exclude_patterns: list[str],
|
|
||||||
metadata_path: pathlib.Path | None,
|
|
||||||
context_profile: str,
|
|
||||||
max_file_bytes: int,
|
|
||||||
) -> int:
|
|
||||||
project_root = project_root.resolve()
|
|
||||||
output_dir = output_dir.resolve()
|
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
metadata = load_metadata(metadata_path)
|
|
||||||
if not project_root.is_dir():
|
|
||||||
manifest = empty_manifest(project_root, review_id, context_profile, source_roots, metadata)
|
|
||||||
manifest["blocking_errors"].append(f"project_root is not a directory: {project_root}")
|
|
||||||
finalize_manifest(manifest)
|
|
||||||
write_outputs(manifest, output_dir)
|
|
||||||
return 2
|
|
||||||
|
|
||||||
normalized_sources = source_roots or ["."]
|
|
||||||
normalized_includes = include_patterns or ["**/*"]
|
|
||||||
manifest = build_manifest(
|
|
||||||
project_root=project_root,
|
|
||||||
review_id=review_id,
|
|
||||||
context_profile=context_profile,
|
|
||||||
source_roots_raw=normalized_sources,
|
|
||||||
include_patterns=normalized_includes,
|
|
||||||
exclude_patterns=exclude_patterns,
|
|
||||||
metadata=metadata,
|
|
||||||
max_file_bytes=max_file_bytes,
|
|
||||||
)
|
|
||||||
write_outputs(manifest, output_dir)
|
|
||||||
return 1 if manifest["blocking_errors"] else 0
|
|
||||||
|
|
||||||
|
|
||||||
def main(argv: list[str] | None = None) -> int:
|
|
||||||
parser = argparse.ArgumentParser(description="Build a file-first review context index.")
|
|
||||||
parser.add_argument("--project-root", required=True, type=pathlib.Path)
|
|
||||||
parser.add_argument("--review-id", required=True)
|
|
||||||
parser.add_argument("--output-dir", required=True, type=pathlib.Path)
|
|
||||||
parser.add_argument("--source-root", action="append", default=[])
|
|
||||||
parser.add_argument("--include-pattern", action="append", default=[])
|
|
||||||
parser.add_argument("--exclude-pattern", action="append", default=[])
|
|
||||||
parser.add_argument("--metadata", type=pathlib.Path)
|
|
||||||
parser.add_argument("--context-profile", default="generic")
|
|
||||||
parser.add_argument("--max-file-bytes", type=int, default=DEFAULT_MAX_FILE_BYTES)
|
|
||||||
args = parser.parse_args(argv)
|
|
||||||
return run(
|
|
||||||
project_root=args.project_root,
|
|
||||||
review_id=args.review_id,
|
|
||||||
output_dir=args.output_dir,
|
|
||||||
source_roots=args.source_root,
|
|
||||||
include_patterns=args.include_pattern,
|
|
||||||
exclude_patterns=args.exclude_pattern,
|
|
||||||
metadata_path=args.metadata,
|
|
||||||
context_profile=args.context_profile,
|
|
||||||
max_file_bytes=args.max_file_bytes,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
|
|
@ -1,234 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
import pathlib
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import tempfile
|
|
||||||
import unittest
|
|
||||||
|
|
||||||
|
|
||||||
SKILL_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
||||||
REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]
|
|
||||||
SCRIPT = SKILL_ROOT / "scripts" / "review_context_builder.py"
|
|
||||||
TMP_ROOT = REPO_ROOT / "tmp" / "review-context-builder-tests"
|
|
||||||
|
|
||||||
|
|
||||||
def run_cli(*args: str) -> subprocess.CompletedProcess[str]:
|
|
||||||
return subprocess.run(
|
|
||||||
[sys.executable, str(SCRIPT), *args],
|
|
||||||
text=True,
|
|
||||||
capture_output=True,
|
|
||||||
check=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def write_text(path: pathlib.Path, text: str = "content") -> pathlib.Path:
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
path.write_text(text, encoding="utf-8")
|
|
||||||
return path
|
|
||||||
|
|
||||||
|
|
||||||
def write_json(path: pathlib.Path, data: object) -> pathlib.Path:
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
||||||
return path
|
|
||||||
|
|
||||||
|
|
||||||
def load_manifest(output_dir: pathlib.Path) -> dict[str, object]:
|
|
||||||
return json.loads((output_dir / "review-file-manifest.json").read_text(encoding="utf-8"))
|
|
||||||
|
|
||||||
|
|
||||||
class ReviewContextBuilderTest(unittest.TestCase):
|
|
||||||
def setUp(self) -> None:
|
|
||||||
TMP_ROOT.mkdir(parents=True, exist_ok=True)
|
|
||||||
self.tmp_dir = tempfile.TemporaryDirectory(dir=TMP_ROOT)
|
|
||||||
self.project_root = pathlib.Path(self.tmp_dir.name) / "project"
|
|
||||||
self.output_dir = pathlib.Path(self.tmp_dir.name) / "outputs"
|
|
||||||
self.project_root.mkdir()
|
|
||||||
|
|
||||||
def tearDown(self) -> None:
|
|
||||||
self.tmp_dir.cleanup()
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def tearDownClass(cls) -> None:
|
|
||||||
if TMP_ROOT.exists():
|
|
||||||
shutil.rmtree(TMP_ROOT)
|
|
||||||
|
|
||||||
def run_builder(self, *extra_args: str) -> tuple[subprocess.CompletedProcess[str], dict[str, object], str]:
|
|
||||||
result = run_cli(
|
|
||||||
"--project-root",
|
|
||||||
str(self.project_root),
|
|
||||||
"--review-id",
|
|
||||||
"round-05-local-pass",
|
|
||||||
"--output-dir",
|
|
||||||
str(self.output_dir),
|
|
||||||
*extra_args,
|
|
||||||
)
|
|
||||||
manifest = load_manifest(self.output_dir)
|
|
||||||
context = (self.output_dir / "review-context.md").read_text(encoding="utf-8")
|
|
||||||
return result, manifest, context
|
|
||||||
|
|
||||||
def test_complete_context_build_indexes_configured_sources(self) -> None:
|
|
||||||
write_text(self.project_root / "README.md", "# Project\n")
|
|
||||||
write_text(self.project_root / "reports" / "gate-run-report.md", "# Gate report\n")
|
|
||||||
write_text(self.project_root / "decisions" / "owner-decision.md", "# Decision\n")
|
|
||||||
metadata = write_json(
|
|
||||||
pathlib.Path(self.tmp_dir.name) / "metadata.json",
|
|
||||||
{
|
|
||||||
"review_goal": "check readiness",
|
|
||||||
"non_goals": ["do not decide approval"],
|
|
||||||
"open_questions": ["is evidence complete?"],
|
|
||||||
"target_reviewer": "local reviewer",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
result, manifest, context = self.run_builder(
|
|
||||||
"--source-root",
|
|
||||||
".",
|
|
||||||
"--include-pattern",
|
|
||||||
"**/*.md",
|
|
||||||
"--metadata",
|
|
||||||
str(metadata),
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual(manifest["review_id"], "round-05-local-pass")
|
|
||||||
self.assertEqual(manifest["context_profile"], "generic")
|
|
||||||
self.assertEqual(
|
|
||||||
manifest["files_included"],
|
|
||||||
[
|
|
||||||
{"path": "README.md", "size_bytes": 10, "source_root": "."},
|
|
||||||
{"path": "decisions/owner-decision.md", "size_bytes": 11, "source_root": "."},
|
|
||||||
{"path": "reports/gate-run-report.md", "size_bytes": 14, "source_root": "."},
|
|
||||||
],
|
|
||||||
)
|
|
||||||
self.assertEqual(manifest["metadata"]["review_goal"], "check readiness")
|
|
||||||
self.assertEqual(manifest["known_non_goals"], ["do not decide approval"])
|
|
||||||
self.assertEqual(manifest["open_questions_for_reviewer_or_agent"], ["is evidence complete?"])
|
|
||||||
self.assertIn("reports/gate-run-report.md", manifest["important_reports"])
|
|
||||||
self.assertIn("decisions/owner-decision.md", manifest["prior_decision_references"])
|
|
||||||
self.assertIn("This context is an index. It does not replace reading source files.", context)
|
|
||||||
|
|
||||||
def test_missing_source_root_is_blocking_but_outputs_manifest(self) -> None:
|
|
||||||
result, manifest, context = self.run_builder("--source-root", "missing")
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 1)
|
|
||||||
self.assertEqual(manifest["machine_readable_summary"]["status"], "FAIL")
|
|
||||||
self.assertEqual(manifest["missing_source_roots"], ["missing"])
|
|
||||||
self.assertIn("Missing source root", manifest["blocking_errors"][0])
|
|
||||||
self.assertIn("missing", context)
|
|
||||||
|
|
||||||
def test_large_files_are_excluded_with_reason(self) -> None:
|
|
||||||
write_text(self.project_root / "small.md", "small")
|
|
||||||
write_text(self.project_root / "large.md", "0123456789")
|
|
||||||
|
|
||||||
result, manifest, _ = self.run_builder(
|
|
||||||
"--source-root",
|
|
||||||
".",
|
|
||||||
"--include-pattern",
|
|
||||||
"**/*.md",
|
|
||||||
"--max-file-bytes",
|
|
||||||
"5",
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual([item["path"] for item in manifest["files_included"]], ["small.md"])
|
|
||||||
self.assertEqual(
|
|
||||||
manifest["files_excluded"],
|
|
||||||
[{"path": "large.md", "reason": "size_exceeds_limit", "size_bytes": 10}],
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_manifest_path_normalization_uses_project_relative_posix_paths(self) -> None:
|
|
||||||
nested = write_text(self.project_root / "docs" / "nested" / "report.md", "report")
|
|
||||||
|
|
||||||
result, manifest, _ = self.run_builder("--source-root", str(nested), "--include-pattern", "**/*.md")
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual(manifest["source_roots"], ["docs/nested/report.md"])
|
|
||||||
self.assertEqual(manifest["files_included"][0]["path"], "docs/nested/report.md")
|
|
||||||
self.assertNotIn("\\", manifest["files_included"][0]["path"])
|
|
||||||
|
|
||||||
def test_utf8_paths_and_chinese_filenames_are_preserved(self) -> None:
|
|
||||||
write_text(self.project_root / "材料" / "审查报告.md", "# 报告\n")
|
|
||||||
|
|
||||||
result, manifest, context = self.run_builder(
|
|
||||||
"--source-root",
|
|
||||||
"材料",
|
|
||||||
"--include-pattern",
|
|
||||||
"**/*.md",
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual(manifest["files_included"][0]["path"], "材料/审查报告.md")
|
|
||||||
self.assertIn("材料/审查报告.md", context)
|
|
||||||
|
|
||||||
def test_stable_output_ordering_across_repeated_runs(self) -> None:
|
|
||||||
write_text(self.project_root / "zeta.md", "z")
|
|
||||||
write_text(self.project_root / "alpha.md", "a")
|
|
||||||
write_text(self.project_root / "nested" / "beta.md", "b")
|
|
||||||
|
|
||||||
first_result, first_manifest, first_context = self.run_builder(
|
|
||||||
"--source-root",
|
|
||||||
".",
|
|
||||||
"--include-pattern",
|
|
||||||
"**/*.md",
|
|
||||||
)
|
|
||||||
second_result, second_manifest, second_context = self.run_builder(
|
|
||||||
"--source-root",
|
|
||||||
".",
|
|
||||||
"--include-pattern",
|
|
||||||
"**/*.md",
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertEqual(first_result.returncode, 0, first_result.stderr)
|
|
||||||
self.assertEqual(second_result.returncode, 0, second_result.stderr)
|
|
||||||
self.assertEqual(first_manifest, second_manifest)
|
|
||||||
self.assertEqual(first_context, second_context)
|
|
||||||
self.assertEqual([item["path"] for item in first_manifest["files_included"]], ["alpha.md", "nested/beta.md", "zeta.md"])
|
|
||||||
|
|
||||||
def test_metadata_passthrough_does_not_add_ccra_fields(self) -> None:
|
|
||||||
metadata = write_json(
|
|
||||||
pathlib.Path(self.tmp_dir.name) / "metadata.json",
|
|
||||||
{"custom_key": "custom value", "round_id": "R05"},
|
|
||||||
)
|
|
||||||
write_text(self.project_root / "note.md", "note")
|
|
||||||
|
|
||||||
result, manifest, _ = self.run_builder(
|
|
||||||
"--source-root",
|
|
||||||
".",
|
|
||||||
"--include-pattern",
|
|
||||||
"**/*.md",
|
|
||||||
"--metadata",
|
|
||||||
str(metadata),
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual(manifest["metadata"], {"custom_key": "custom value", "round_id": "R05"})
|
|
||||||
self.assertNotIn("review_round", manifest)
|
|
||||||
self.assertNotIn("local_pass", manifest)
|
|
||||||
|
|
||||||
def test_exclude_patterns_remove_matching_files(self) -> None:
|
|
||||||
write_text(self.project_root / "include.md", "include")
|
|
||||||
write_text(self.project_root / "scratch" / "exclude.md", "exclude")
|
|
||||||
|
|
||||||
result, manifest, _ = self.run_builder(
|
|
||||||
"--source-root",
|
|
||||||
".",
|
|
||||||
"--include-pattern",
|
|
||||||
"**/*.md",
|
|
||||||
"--exclude-pattern",
|
|
||||||
"scratch/**",
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual([item["path"] for item in manifest["files_included"]], ["include.md"])
|
|
||||||
self.assertEqual(
|
|
||||||
manifest["files_excluded"],
|
|
||||||
[{"path": "scratch/exclude.md", "reason": "exclude_pattern"}],
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
unittest.main()
|
|
||||||
|
|
@ -1,43 +0,0 @@
|
||||||
# Routing Behavior Diff Audit
|
|
||||||
|
|
||||||
Source Skill for deterministic before/after routing result comparison.
|
|
||||||
|
|
||||||
The canonical entry point is `SKILL.md`; the deterministic diff engine is in `scripts/routing_behavior_diff_audit.py`.
|
|
||||||
|
|
||||||
## Original Source
|
|
||||||
|
|
||||||
```text
|
|
||||||
Original source: C:\Users\wangq\Documents\Codex\ccpe-system\requirements\skills-vault\2026-06-19-routing-behavior-diff-audit.md
|
|
||||||
Migration date: 2026-06-19
|
|
||||||
Migration status: first public Skill implementation
|
|
||||||
Behavior preserved: deterministic routing-result diff only; no selector edits, expected-label updates, or patch-closure judgment
|
|
||||||
Known gaps: Markdown support is limited to fenced JSON/YAML blocks; product-specific meaning remains outside this Skill
|
|
||||||
```
|
|
||||||
|
|
||||||
## Layout
|
|
||||||
|
|
||||||
```text
|
|
||||||
SKILL.md
|
|
||||||
README.md
|
|
||||||
agents/openai.yaml
|
|
||||||
fixtures/before.example.json
|
|
||||||
fixtures/after.example.json
|
|
||||||
scripts/routing_behavior_diff_audit.py
|
|
||||||
tests/test_routing_behavior_diff_audit.py
|
|
||||||
```
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
conda run -n skills-vault python .\scripts\routing_behavior_diff_audit.py `
|
|
||||||
--before-results C:\path\before.json `
|
|
||||||
--after-results C:\path\after.json `
|
|
||||||
--output-dir C:\path\helper-outputs `
|
|
||||||
--targeted-case CASE-001
|
|
||||||
```
|
|
||||||
|
|
||||||
## Tests
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
conda run -n skills-vault python -B -m unittest discover -s skills/routing-behavior-diff-audit/tests -v
|
|
||||||
```
|
|
||||||
|
|
@ -1,92 +0,0 @@
|
||||||
---
|
|
||||||
name: routing-behavior-diff-audit
|
|
||||||
description: Use when Codex needs to compare before/after selector, routing, QPI, classifier, rules-engine, triage, assignment, or model-governance outputs to identify targeted route changes, collateral non-target changes, no-call changes, missing cases, and new or resolved expected-route mismatches before review, release, handoff, CCRA/local review, or regression closeout.
|
|
||||||
---
|
|
||||||
|
|
||||||
# Routing Behavior Diff Audit
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
Run a deterministic before/after behavior diff for rules-based routing outputs. Use the bundled script to compare case IDs, routes, optional expected routes, missing cases, no-call transitions, and targeted versus non-target changes.
|
|
||||||
|
|
||||||
This Skill reports observed differences only. It must not edit routing rules, change regression cases, update expected labels, infer product meaning, or decide whether a patch should close.
|
|
||||||
|
|
||||||
## Command
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
conda run -n skills-vault python .\scripts\routing_behavior_diff_audit.py `
|
|
||||||
--before-results C:\path\before-results `
|
|
||||||
--after-results C:\path\after-results `
|
|
||||||
--output-dir C:\path\review-run `
|
|
||||||
--case-id-field case_id `
|
|
||||||
--route-field route `
|
|
||||||
--expected-route-field expected_route `
|
|
||||||
--targeted-case CASE-001
|
|
||||||
```
|
|
||||||
|
|
||||||
When installed under `C:\Users\wangq\.agents\skills\routing-behavior-diff-audit`, run from that Skill directory or use the installed script path.
|
|
||||||
|
|
||||||
## Inputs
|
|
||||||
|
|
||||||
- `--before-results`: JSON, JSONL, or Markdown file, or a directory containing those files.
|
|
||||||
- `--after-results`: JSON, JSONL, or Markdown file, or a directory containing those files.
|
|
||||||
- `--output-dir`: directory where reports are written.
|
|
||||||
- `--case-id-field`: field containing the stable case ID. Default: `case_id`.
|
|
||||||
- `--route-field`: field containing the routing decision. Default: `route`.
|
|
||||||
- `--expected-route-field`: optional field containing the expected route for failure comparison.
|
|
||||||
- `--targeted-case`: repeatable case ID that the patch intentionally targets.
|
|
||||||
- `--targeted-cases-file`: optional JSON/YAML list, or object with `targeted_cases`, `case_ids`, or `cases`.
|
|
||||||
|
|
||||||
JSON is the stable input target. Directory inputs are scanned recursively for `.json`, `.jsonl`, `.md`, and `.markdown`; Markdown support expects a fenced JSON or YAML block.
|
|
||||||
|
|
||||||
## Outputs
|
|
||||||
|
|
||||||
The script writes under `output_dir`:
|
|
||||||
|
|
||||||
```text
|
|
||||||
routing-behavior-diff.md
|
|
||||||
routing-behavior-diff.json
|
|
||||||
```
|
|
||||||
|
|
||||||
The JSON report includes:
|
|
||||||
|
|
||||||
- `total_cases_compared`
|
|
||||||
- `unchanged_cases`
|
|
||||||
- `changed_cases`
|
|
||||||
- `targeted_changes`
|
|
||||||
- `non_target_changes`
|
|
||||||
- `new_failures`
|
|
||||||
- `resolved_failures`
|
|
||||||
- `missing_before_cases`
|
|
||||||
- `missing_after_cases`
|
|
||||||
- `diff_table`
|
|
||||||
- `machine_readable_summary`
|
|
||||||
|
|
||||||
`machine_readable_summary.status` is `PASS`, `ATTENTION`, or `FAIL` for automation routing only. It is not a patch approval, owner approval, CCRA decision, or release decision.
|
|
||||||
|
|
||||||
## Comparison Rules
|
|
||||||
|
|
||||||
- Compare only case IDs present in both before and after for `total_cases_compared`.
|
|
||||||
- Classify after-only IDs as `missing_before_cases`.
|
|
||||||
- Classify before-only IDs as `missing_after_cases`.
|
|
||||||
- Treat a common case as changed when the route, expected route, no-call status, or expected-route failure status changes.
|
|
||||||
- Classify changed cases listed by `--targeted-case` or `--targeted-cases-file` as `targeted_changes`.
|
|
||||||
- Classify all other changed cases as `non_target_changes`.
|
|
||||||
- Compute `new_failures` and `resolved_failures` only when `--expected-route-field` is supplied and present in the records.
|
|
||||||
- Treat empty, null-like, and common no-call spellings such as `no_call`, `no-call`, and `no call` as no-call values.
|
|
||||||
|
|
||||||
## Safety
|
|
||||||
|
|
||||||
- Read only configured before/after result paths and optional targeted-cases file.
|
|
||||||
- Write only `routing-behavior-diff.md` and `routing-behavior-diff.json` under `output_dir`.
|
|
||||||
- Do not edit selectors, routing rules, regression cases, expected labels, review bundles, model cards, or governance artifacts.
|
|
||||||
- Do not decide acceptance, lifecycle status, patch closure, release readiness, or owner approval.
|
|
||||||
|
|
||||||
## Validation
|
|
||||||
|
|
||||||
After changing diff behavior, run:
|
|
||||||
|
|
||||||
```powershell
|
|
||||||
conda run -n skills-vault python -B -m unittest discover -s skills/routing-behavior-diff-audit/tests -v
|
|
||||||
conda run -n skills-vault python -B C:\Users\wangq\.codex\skills\.system\skill-creator\scripts\quick_validate.py skills/routing-behavior-diff-audit
|
|
||||||
```
|
|
||||||
|
|
@ -1,4 +0,0 @@
|
||||||
interface:
|
|
||||||
display_name: "Routing Behavior Diff Audit"
|
|
||||||
short_description: "Compare before/after routing outputs"
|
|
||||||
default_prompt: "Use $routing-behavior-diff-audit to compare before and after routing results and write routing-behavior-diff reports."
|
|
||||||
|
|
@ -1,12 +0,0 @@
|
||||||
[
|
|
||||||
{
|
|
||||||
"case_id": "CASE-001",
|
|
||||||
"route": "qpi-revised",
|
|
||||||
"expected_route": "qpi-revised"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"case_id": "CASE-002",
|
|
||||||
"route": "review",
|
|
||||||
"expected_route": "review"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
|
|
@ -1,12 +0,0 @@
|
||||||
[
|
|
||||||
{
|
|
||||||
"case_id": "CASE-001",
|
|
||||||
"route": "qpi",
|
|
||||||
"expected_route": "qpi"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"case_id": "CASE-002",
|
|
||||||
"route": "no_call",
|
|
||||||
"expected_route": "review"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
|
|
@ -1,491 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
import pathlib
|
|
||||||
import re
|
|
||||||
import sys
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
|
|
||||||
RESULT_KEYS = ("results", "cases", "records", "items", "diff_table")
|
|
||||||
SUPPORTED_SUFFIXES = {".json", ".jsonl", ".md", ".markdown"}
|
|
||||||
NO_CALL_VALUES = {"", "none", "null", "no_call", "no-call", "no call", "nocall", "no decision"}
|
|
||||||
|
|
||||||
|
|
||||||
def main(argv: list[str] | None = None) -> int:
|
|
||||||
args = parse_args(argv)
|
|
||||||
targeted_cases, targeted_errors = load_targeted_cases(args.targeted_case, args.targeted_cases_file)
|
|
||||||
before_cases, before_errors = load_case_map(
|
|
||||||
args.before_results,
|
|
||||||
args.case_id_field,
|
|
||||||
args.route_field,
|
|
||||||
args.expected_route_field,
|
|
||||||
"before",
|
|
||||||
)
|
|
||||||
after_cases, after_errors = load_case_map(
|
|
||||||
args.after_results,
|
|
||||||
args.case_id_field,
|
|
||||||
args.route_field,
|
|
||||||
args.expected_route_field,
|
|
||||||
"after",
|
|
||||||
)
|
|
||||||
blocking_errors = [*targeted_errors, *before_errors, *after_errors]
|
|
||||||
|
|
||||||
if blocking_errors:
|
|
||||||
summary = empty_summary(blocking_errors)
|
|
||||||
write_reports(args.output_dir, summary)
|
|
||||||
print(f"routing-behavior-diff-audit: {len(blocking_errors)} blocking error(s)", file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
summary = compare_cases(before_cases, after_cases, targeted_cases)
|
|
||||||
write_reports(args.output_dir, summary)
|
|
||||||
print(str(args.output_dir / "routing-behavior-diff.json"))
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
def parse_args(argv: list[str] | None) -> argparse.Namespace:
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="Compare before/after rules-based routing outputs and write deterministic diff reports.",
|
|
||||||
)
|
|
||||||
parser.add_argument("--before-results", required=True, type=pathlib.Path)
|
|
||||||
parser.add_argument("--after-results", required=True, type=pathlib.Path)
|
|
||||||
parser.add_argument("--output-dir", required=True, type=pathlib.Path)
|
|
||||||
parser.add_argument("--case-id-field", default="case_id")
|
|
||||||
parser.add_argument("--route-field", default="route")
|
|
||||||
parser.add_argument("--expected-route-field", default=None)
|
|
||||||
parser.add_argument("--targeted-case", action="append", default=[])
|
|
||||||
parser.add_argument("--targeted-cases-file", type=pathlib.Path)
|
|
||||||
return parser.parse_args(argv)
|
|
||||||
|
|
||||||
|
|
||||||
def load_targeted_cases(raw_cases: list[str], cases_file: pathlib.Path | None) -> tuple[set[str], list[str]]:
|
|
||||||
targeted = {str(case_id) for case_id in raw_cases}
|
|
||||||
errors: list[str] = []
|
|
||||||
if not cases_file:
|
|
||||||
return targeted, errors
|
|
||||||
if not cases_file.exists():
|
|
||||||
return targeted, [f"Targeted cases file not found: {cases_file}"]
|
|
||||||
|
|
||||||
try:
|
|
||||||
loaded = parse_file(cases_file)
|
|
||||||
except ValueError as exc:
|
|
||||||
return targeted, [f"Unable to parse targeted cases file {cases_file}: {exc}"]
|
|
||||||
|
|
||||||
candidate: Any = loaded
|
|
||||||
if isinstance(candidate, dict):
|
|
||||||
for key in ("targeted_cases", "case_ids", "cases"):
|
|
||||||
if key in candidate:
|
|
||||||
candidate = candidate[key]
|
|
||||||
break
|
|
||||||
if not isinstance(candidate, list):
|
|
||||||
return targeted, [f"Targeted cases file must contain a list or object with targeted_cases/case_ids/cases: {cases_file}"]
|
|
||||||
|
|
||||||
for item in candidate:
|
|
||||||
if isinstance(item, dict):
|
|
||||||
for key in ("case_id", "id"):
|
|
||||||
if key in item:
|
|
||||||
targeted.add(str(item[key]))
|
|
||||||
break
|
|
||||||
else:
|
|
||||||
targeted.add(str(item))
|
|
||||||
return targeted, errors
|
|
||||||
|
|
||||||
|
|
||||||
def load_case_map(
|
|
||||||
root: pathlib.Path,
|
|
||||||
case_id_field: str,
|
|
||||||
route_field: str,
|
|
||||||
expected_route_field: str | None,
|
|
||||||
side: str,
|
|
||||||
) -> tuple[dict[str, dict[str, Any]], list[str]]:
|
|
||||||
errors: list[str] = []
|
|
||||||
if not root.exists():
|
|
||||||
return {}, [f"{side} results path not found: {root}"]
|
|
||||||
|
|
||||||
files = collect_input_files(root)
|
|
||||||
if not files:
|
|
||||||
return {}, [f"{side} results path contains no supported files: {root}"]
|
|
||||||
|
|
||||||
cases: dict[str, dict[str, Any]] = {}
|
|
||||||
for path in files:
|
|
||||||
try:
|
|
||||||
parsed = parse_file(path)
|
|
||||||
except ValueError as exc:
|
|
||||||
errors.append(f"Unable to parse {side} file {path}: {exc}")
|
|
||||||
continue
|
|
||||||
|
|
||||||
records = extract_records(parsed)
|
|
||||||
if not records:
|
|
||||||
errors.append(f"No records found in {side} file: {path}")
|
|
||||||
continue
|
|
||||||
|
|
||||||
for record in records:
|
|
||||||
if not isinstance(record, dict):
|
|
||||||
errors.append(f"Record in {side} file is not an object: {path}")
|
|
||||||
continue
|
|
||||||
if case_id_field not in record:
|
|
||||||
errors.append(f"Record missing case id field '{case_id_field}' in {side} file: {path}")
|
|
||||||
continue
|
|
||||||
case_id = stable_string(record[case_id_field])
|
|
||||||
if case_id in cases:
|
|
||||||
errors.append(f"Duplicate case id '{case_id}' in {side} inputs")
|
|
||||||
continue
|
|
||||||
cases[case_id] = {
|
|
||||||
"case_id": case_id,
|
|
||||||
"route": normalize_value(record.get(route_field)),
|
|
||||||
"expected_route": normalize_value(record.get(expected_route_field)) if expected_route_field else None,
|
|
||||||
"source_file": str(path),
|
|
||||||
}
|
|
||||||
return cases, errors
|
|
||||||
|
|
||||||
|
|
||||||
def collect_input_files(root: pathlib.Path) -> list[pathlib.Path]:
|
|
||||||
if root.is_file():
|
|
||||||
return [root] if root.suffix.lower() in SUPPORTED_SUFFIXES else []
|
|
||||||
return sorted(path for path in root.rglob("*") if path.is_file() and path.suffix.lower() in SUPPORTED_SUFFIXES)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_file(path: pathlib.Path) -> Any:
|
|
||||||
suffix = path.suffix.lower()
|
|
||||||
text = path.read_text(encoding="utf-8")
|
|
||||||
if suffix == ".json":
|
|
||||||
return json.loads(text)
|
|
||||||
if suffix == ".jsonl":
|
|
||||||
records = []
|
|
||||||
for line_number, line in enumerate(text.splitlines(), start=1):
|
|
||||||
stripped = line.strip()
|
|
||||||
if not stripped:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
records.append(json.loads(stripped))
|
|
||||||
except json.JSONDecodeError as exc:
|
|
||||||
raise ValueError(f"invalid JSONL on line {line_number}: {exc}") from exc
|
|
||||||
return records
|
|
||||||
if suffix in {".md", ".markdown"}:
|
|
||||||
return parse_markdown_summary(text, path)
|
|
||||||
raise ValueError(f"unsupported suffix: {suffix}")
|
|
||||||
|
|
||||||
|
|
||||||
def parse_markdown_summary(text: str, path: pathlib.Path) -> Any:
|
|
||||||
fences = re.findall(r"```(?:json|yaml|yml)?\s*\n(.*?)\n```", text, flags=re.DOTALL | re.IGNORECASE)
|
|
||||||
errors: list[str] = []
|
|
||||||
for fence in fences:
|
|
||||||
try:
|
|
||||||
return json.loads(fence)
|
|
||||||
except json.JSONDecodeError as exc:
|
|
||||||
errors.append(f"json: {exc}")
|
|
||||||
yaml_loaded = try_load_yaml(fence)
|
|
||||||
if yaml_loaded is not None:
|
|
||||||
return yaml_loaded
|
|
||||||
raise ValueError(f"no parseable JSON/YAML fenced block found in Markdown summary {path}; {'; '.join(errors)}")
|
|
||||||
|
|
||||||
|
|
||||||
def try_load_yaml(text: str) -> Any:
|
|
||||||
try:
|
|
||||||
import yaml # type: ignore[import-not-found]
|
|
||||||
except ImportError:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
return yaml.safe_load(text)
|
|
||||||
except Exception:
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def extract_records(parsed: Any) -> list[Any]:
|
|
||||||
if isinstance(parsed, list):
|
|
||||||
return parsed
|
|
||||||
if isinstance(parsed, dict):
|
|
||||||
for key in RESULT_KEYS:
|
|
||||||
value = parsed.get(key)
|
|
||||||
if isinstance(value, list):
|
|
||||||
return value
|
|
||||||
if "case_id" in parsed or "id" in parsed:
|
|
||||||
return [parsed]
|
|
||||||
return []
|
|
||||||
|
|
||||||
|
|
||||||
def compare_cases(before_cases: dict[str, dict[str, Any]], after_cases: dict[str, dict[str, Any]], targeted_cases: set[str]) -> dict[str, Any]:
|
|
||||||
before_ids = set(before_cases)
|
|
||||||
after_ids = set(after_cases)
|
|
||||||
common_ids = sorted(before_ids & after_ids)
|
|
||||||
missing_before = sorted(after_ids - before_ids)
|
|
||||||
missing_after = sorted(before_ids - after_ids)
|
|
||||||
|
|
||||||
unchanged: list[str] = []
|
|
||||||
changed: list[str] = []
|
|
||||||
targeted_changes: list[str] = []
|
|
||||||
non_target_changes: list[str] = []
|
|
||||||
new_failures: list[str] = []
|
|
||||||
resolved_failures: list[str] = []
|
|
||||||
diff_table: list[dict[str, Any]] = []
|
|
||||||
|
|
||||||
for case_id in common_ids:
|
|
||||||
before = before_cases[case_id]
|
|
||||||
after = after_cases[case_id]
|
|
||||||
before_failure = is_failure(before)
|
|
||||||
after_failure = is_failure(after)
|
|
||||||
route_changed = before["route"] != after["route"]
|
|
||||||
expected_changed = before["expected_route"] != after["expected_route"]
|
|
||||||
failure_status_changed = before_failure != after_failure
|
|
||||||
no_call_changed = is_no_call(before["route"]) != is_no_call(after["route"])
|
|
||||||
row_changed = route_changed or expected_changed or failure_status_changed or no_call_changed
|
|
||||||
change_type = build_change_type(route_changed, expected_changed, failure_status_changed, no_call_changed)
|
|
||||||
|
|
||||||
if row_changed:
|
|
||||||
changed.append(case_id)
|
|
||||||
if case_id in targeted_cases:
|
|
||||||
targeted_changes.append(case_id)
|
|
||||||
else:
|
|
||||||
non_target_changes.append(case_id)
|
|
||||||
else:
|
|
||||||
unchanged.append(case_id)
|
|
||||||
|
|
||||||
if not before_failure and after_failure:
|
|
||||||
new_failures.append(case_id)
|
|
||||||
if before_failure and not after_failure:
|
|
||||||
resolved_failures.append(case_id)
|
|
||||||
|
|
||||||
diff_table.append(
|
|
||||||
{
|
|
||||||
"case_id": case_id,
|
|
||||||
"change_type": change_type,
|
|
||||||
"targeted": case_id in targeted_cases,
|
|
||||||
"before_route": before["route"],
|
|
||||||
"after_route": after["route"],
|
|
||||||
"before_expected_route": before["expected_route"],
|
|
||||||
"after_expected_route": after["expected_route"],
|
|
||||||
"route_changed": route_changed,
|
|
||||||
"expected_route_changed": expected_changed,
|
|
||||||
"before_failure": before_failure,
|
|
||||||
"after_failure": after_failure,
|
|
||||||
"failure_status_changed": failure_status_changed,
|
|
||||||
"no_call_changed": no_call_changed,
|
|
||||||
"before_source_file": before["source_file"],
|
|
||||||
"after_source_file": after["source_file"],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
for case_id in missing_before:
|
|
||||||
after = after_cases[case_id]
|
|
||||||
diff_table.append(
|
|
||||||
{
|
|
||||||
"case_id": case_id,
|
|
||||||
"change_type": "missing_before",
|
|
||||||
"targeted": case_id in targeted_cases,
|
|
||||||
"before_route": None,
|
|
||||||
"after_route": after["route"],
|
|
||||||
"before_expected_route": None,
|
|
||||||
"after_expected_route": after["expected_route"],
|
|
||||||
"route_changed": True,
|
|
||||||
"expected_route_changed": after["expected_route"] is not None,
|
|
||||||
"before_failure": None,
|
|
||||||
"after_failure": is_failure(after),
|
|
||||||
"failure_status_changed": None,
|
|
||||||
"no_call_changed": None,
|
|
||||||
"before_source_file": None,
|
|
||||||
"after_source_file": after["source_file"],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
for case_id in missing_after:
|
|
||||||
before = before_cases[case_id]
|
|
||||||
diff_table.append(
|
|
||||||
{
|
|
||||||
"case_id": case_id,
|
|
||||||
"change_type": "missing_after",
|
|
||||||
"targeted": case_id in targeted_cases,
|
|
||||||
"before_route": before["route"],
|
|
||||||
"after_route": None,
|
|
||||||
"before_expected_route": before["expected_route"],
|
|
||||||
"after_expected_route": None,
|
|
||||||
"route_changed": True,
|
|
||||||
"expected_route_changed": before["expected_route"] is not None,
|
|
||||||
"before_failure": is_failure(before),
|
|
||||||
"after_failure": None,
|
|
||||||
"failure_status_changed": None,
|
|
||||||
"no_call_changed": None,
|
|
||||||
"before_source_file": before["source_file"],
|
|
||||||
"after_source_file": None,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
status = "ATTENTION" if non_target_changes or missing_before or missing_after or new_failures else "PASS"
|
|
||||||
return {
|
|
||||||
"total_cases_compared": len(common_ids),
|
|
||||||
"unchanged_cases": unchanged,
|
|
||||||
"changed_cases": changed,
|
|
||||||
"targeted_changes": targeted_changes,
|
|
||||||
"non_target_changes": non_target_changes,
|
|
||||||
"new_failures": new_failures,
|
|
||||||
"resolved_failures": resolved_failures,
|
|
||||||
"missing_before_cases": missing_before,
|
|
||||||
"missing_after_cases": missing_after,
|
|
||||||
"diff_table": sorted(diff_table, key=lambda row: row["case_id"]),
|
|
||||||
"blocking_errors": [],
|
|
||||||
"machine_readable_summary": {
|
|
||||||
"status": status,
|
|
||||||
"total_cases_compared": len(common_ids),
|
|
||||||
"changed_count": len(changed),
|
|
||||||
"targeted_change_count": len(targeted_changes),
|
|
||||||
"non_target_change_count": len(non_target_changes),
|
|
||||||
"new_failure_count": len(new_failures),
|
|
||||||
"resolved_failure_count": len(resolved_failures),
|
|
||||||
"missing_before_count": len(missing_before),
|
|
||||||
"missing_after_count": len(missing_after),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def build_change_type(route_changed: bool, expected_changed: bool, failure_status_changed: bool, no_call_changed: bool) -> str:
|
|
||||||
labels: list[str] = []
|
|
||||||
if route_changed:
|
|
||||||
labels.append("route_changed")
|
|
||||||
if expected_changed:
|
|
||||||
labels.append("expected_route_changed")
|
|
||||||
if failure_status_changed:
|
|
||||||
labels.append("failure_status_changed")
|
|
||||||
if no_call_changed:
|
|
||||||
labels.append("no_call_changed")
|
|
||||||
return "+".join(labels) if labels else "unchanged"
|
|
||||||
|
|
||||||
|
|
||||||
def is_failure(case: dict[str, Any]) -> bool:
|
|
||||||
expected = case.get("expected_route")
|
|
||||||
if expected is None:
|
|
||||||
return False
|
|
||||||
return case.get("route") != expected
|
|
||||||
|
|
||||||
|
|
||||||
def is_no_call(value: Any) -> bool:
|
|
||||||
if value is None:
|
|
||||||
return True
|
|
||||||
return str(value).strip().casefold() in NO_CALL_VALUES
|
|
||||||
|
|
||||||
|
|
||||||
def normalize_value(value: Any) -> str | None:
|
|
||||||
if value is None:
|
|
||||||
return None
|
|
||||||
return stable_string(value)
|
|
||||||
|
|
||||||
|
|
||||||
def stable_string(value: Any) -> str:
|
|
||||||
if isinstance(value, str):
|
|
||||||
return value
|
|
||||||
return json.dumps(value, ensure_ascii=False, sort_keys=True)
|
|
||||||
|
|
||||||
|
|
||||||
def empty_summary(blocking_errors: list[str]) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"total_cases_compared": 0,
|
|
||||||
"unchanged_cases": [],
|
|
||||||
"changed_cases": [],
|
|
||||||
"targeted_changes": [],
|
|
||||||
"non_target_changes": [],
|
|
||||||
"new_failures": [],
|
|
||||||
"resolved_failures": [],
|
|
||||||
"missing_before_cases": [],
|
|
||||||
"missing_after_cases": [],
|
|
||||||
"diff_table": [],
|
|
||||||
"blocking_errors": blocking_errors,
|
|
||||||
"machine_readable_summary": {
|
|
||||||
"status": "FAIL",
|
|
||||||
"blocking_error_count": len(blocking_errors),
|
|
||||||
"total_cases_compared": 0,
|
|
||||||
"changed_count": 0,
|
|
||||||
"targeted_change_count": 0,
|
|
||||||
"non_target_change_count": 0,
|
|
||||||
"new_failure_count": 0,
|
|
||||||
"resolved_failure_count": 0,
|
|
||||||
"missing_before_count": 0,
|
|
||||||
"missing_after_count": 0,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def write_reports(output_dir: pathlib.Path, summary: dict[str, Any]) -> None:
|
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
(output_dir / "routing-behavior-diff.json").write_text(
|
|
||||||
json.dumps(summary, ensure_ascii=False, indent=2) + "\n",
|
|
||||||
encoding="utf-8",
|
|
||||||
)
|
|
||||||
(output_dir / "routing-behavior-diff.md").write_text(render_markdown(summary), encoding="utf-8")
|
|
||||||
|
|
||||||
|
|
||||||
def render_markdown(summary: dict[str, Any]) -> str:
|
|
||||||
machine = summary["machine_readable_summary"]
|
|
||||||
lines = [
|
|
||||||
"# Routing Behavior Diff",
|
|
||||||
"",
|
|
||||||
f"Status: `{machine['status']}`",
|
|
||||||
"",
|
|
||||||
"## Summary",
|
|
||||||
"",
|
|
||||||
f"- Total cases compared: {summary['total_cases_compared']}",
|
|
||||||
f"- Unchanged cases: {len(summary['unchanged_cases'])}",
|
|
||||||
f"- Changed cases: {len(summary['changed_cases'])}",
|
|
||||||
f"- Targeted changes: {len(summary['targeted_changes'])}",
|
|
||||||
f"- Non-target changes: {len(summary['non_target_changes'])}",
|
|
||||||
f"- New failures: {len(summary['new_failures'])}",
|
|
||||||
f"- Resolved failures: {len(summary['resolved_failures'])}",
|
|
||||||
f"- Missing before cases: {len(summary['missing_before_cases'])}",
|
|
||||||
f"- Missing after cases: {len(summary['missing_after_cases'])}",
|
|
||||||
"",
|
|
||||||
]
|
|
||||||
if summary["blocking_errors"]:
|
|
||||||
lines.extend(["## Blocking Errors", ""])
|
|
||||||
lines.extend(f"- {error}" for error in summary["blocking_errors"])
|
|
||||||
lines.append("")
|
|
||||||
|
|
||||||
lines.extend(
|
|
||||||
[
|
|
||||||
"## Diff Table",
|
|
||||||
"",
|
|
||||||
"| Case ID | Change Type | Targeted | Before Route | After Route | Before Failure | After Failure | No-Call Changed |",
|
|
||||||
"| --- | --- | --- | --- | --- | --- | --- | --- |",
|
|
||||||
]
|
|
||||||
)
|
|
||||||
if summary["diff_table"]:
|
|
||||||
for row in summary["diff_table"]:
|
|
||||||
lines.append(
|
|
||||||
"| "
|
|
||||||
+ " | ".join(
|
|
||||||
markdown_cell(row[key])
|
|
||||||
for key in (
|
|
||||||
"case_id",
|
|
||||||
"change_type",
|
|
||||||
"targeted",
|
|
||||||
"before_route",
|
|
||||||
"after_route",
|
|
||||||
"before_failure",
|
|
||||||
"after_failure",
|
|
||||||
"no_call_changed",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
+ " |"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
lines.append("| _none_ | _none_ | _none_ | _none_ | _none_ | _none_ | _none_ | _none_ |")
|
|
||||||
|
|
||||||
lines.extend(
|
|
||||||
[
|
|
||||||
"",
|
|
||||||
"## Machine Readable Summary",
|
|
||||||
"",
|
|
||||||
"```json",
|
|
||||||
json.dumps(machine, ensure_ascii=False, indent=2),
|
|
||||||
"```",
|
|
||||||
"",
|
|
||||||
]
|
|
||||||
)
|
|
||||||
return "\n".join(lines)
|
|
||||||
|
|
||||||
|
|
||||||
def markdown_cell(value: Any) -> str:
|
|
||||||
if value is None:
|
|
||||||
return ""
|
|
||||||
return str(value).replace("|", "\\|").replace("\n", " ")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
|
|
@ -1,210 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
import pathlib
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import tempfile
|
|
||||||
import unittest
|
|
||||||
|
|
||||||
|
|
||||||
SKILL_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
||||||
REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]
|
|
||||||
SCRIPT = SKILL_ROOT / "scripts" / "routing_behavior_diff_audit.py"
|
|
||||||
TMP_ROOT = REPO_ROOT / "tmp" / "routing-behavior-diff-audit-tests"
|
|
||||||
|
|
||||||
|
|
||||||
def run_cli(*args: str) -> subprocess.CompletedProcess[str]:
|
|
||||||
return subprocess.run(
|
|
||||||
[sys.executable, str(SCRIPT), *args],
|
|
||||||
text=True,
|
|
||||||
capture_output=True,
|
|
||||||
check=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def write_json(path: pathlib.Path, data: object) -> pathlib.Path:
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
||||||
return path
|
|
||||||
|
|
||||||
|
|
||||||
def load_summary(output_dir: pathlib.Path) -> dict[str, object]:
|
|
||||||
return json.loads((output_dir / "routing-behavior-diff.json").read_text(encoding="utf-8"))
|
|
||||||
|
|
||||||
|
|
||||||
class RoutingBehaviorDiffAuditTest(unittest.TestCase):
|
|
||||||
def setUp(self) -> None:
|
|
||||||
TMP_ROOT.mkdir(parents=True, exist_ok=True)
|
|
||||||
self.tmp_dir = tempfile.TemporaryDirectory(dir=TMP_ROOT)
|
|
||||||
self.root = pathlib.Path(self.tmp_dir.name)
|
|
||||||
self.before = self.root / "before.json"
|
|
||||||
self.after = self.root / "after.json"
|
|
||||||
self.output_dir = self.root / "outputs"
|
|
||||||
|
|
||||||
def tearDown(self) -> None:
|
|
||||||
self.tmp_dir.cleanup()
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def tearDownClass(cls) -> None:
|
|
||||||
if TMP_ROOT.exists():
|
|
||||||
shutil.rmtree(TMP_ROOT)
|
|
||||||
|
|
||||||
def run_audit(self, *extra_args: str) -> tuple[subprocess.CompletedProcess[str], dict[str, object], str]:
|
|
||||||
result = run_cli(
|
|
||||||
"--before-results",
|
|
||||||
str(self.before),
|
|
||||||
"--after-results",
|
|
||||||
str(self.after),
|
|
||||||
"--output-dir",
|
|
||||||
str(self.output_dir),
|
|
||||||
"--case-id-field",
|
|
||||||
"case_id",
|
|
||||||
"--route-field",
|
|
||||||
"route",
|
|
||||||
"--expected-route-field",
|
|
||||||
"expected_route",
|
|
||||||
*extra_args,
|
|
||||||
)
|
|
||||||
summary = load_summary(self.output_dir)
|
|
||||||
markdown = (self.output_dir / "routing-behavior-diff.md").read_text(encoding="utf-8")
|
|
||||||
return result, summary, markdown
|
|
||||||
|
|
||||||
def test_unchanged_full_set_reports_all_cases_unchanged(self) -> None:
|
|
||||||
rows = [
|
|
||||||
{"case_id": "case-1", "route": "alpha", "expected_route": "alpha"},
|
|
||||||
{"case_id": "case-2", "route": "beta", "expected_route": "beta"},
|
|
||||||
]
|
|
||||||
write_json(self.before, rows)
|
|
||||||
write_json(self.after, rows)
|
|
||||||
|
|
||||||
result, summary, markdown = self.run_audit()
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual(summary["total_cases_compared"], 2)
|
|
||||||
self.assertEqual(summary["unchanged_cases"], ["case-1", "case-2"])
|
|
||||||
self.assertEqual(summary["changed_cases"], [])
|
|
||||||
self.assertEqual(summary["non_target_changes"], [])
|
|
||||||
self.assertEqual(summary["machine_readable_summary"]["status"], "PASS")
|
|
||||||
self.assertIn("case-1", markdown)
|
|
||||||
|
|
||||||
def test_targeted_route_changes_are_classified_separately(self) -> None:
|
|
||||||
write_json(self.before, [{"case_id": "case-1", "route": "alpha", "expected_route": "beta"}])
|
|
||||||
write_json(self.after, [{"case_id": "case-1", "route": "beta", "expected_route": "beta"}])
|
|
||||||
|
|
||||||
result, summary, _ = self.run_audit("--targeted-case", "case-1")
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual(summary["changed_cases"], ["case-1"])
|
|
||||||
self.assertEqual(summary["targeted_changes"], ["case-1"])
|
|
||||||
self.assertEqual(summary["non_target_changes"], [])
|
|
||||||
self.assertEqual(summary["resolved_failures"], ["case-1"])
|
|
||||||
|
|
||||||
def test_non_target_collateral_changes_are_blocker_candidates(self) -> None:
|
|
||||||
write_json(
|
|
||||||
self.before,
|
|
||||||
[
|
|
||||||
{"case_id": "case-1", "route": "alpha"},
|
|
||||||
{"case_id": "case-2", "route": "beta"},
|
|
||||||
],
|
|
||||||
)
|
|
||||||
write_json(
|
|
||||||
self.after,
|
|
||||||
[
|
|
||||||
{"case_id": "case-1", "route": "alpha"},
|
|
||||||
{"case_id": "case-2", "route": "gamma"},
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
result, summary, _ = self.run_audit("--targeted-case", "case-1")
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual(summary["changed_cases"], ["case-2"])
|
|
||||||
self.assertEqual(summary["targeted_changes"], [])
|
|
||||||
self.assertEqual(summary["non_target_changes"], ["case-2"])
|
|
||||||
self.assertEqual(summary["machine_readable_summary"]["status"], "ATTENTION")
|
|
||||||
|
|
||||||
def test_case_missing_in_before_is_reported(self) -> None:
|
|
||||||
write_json(self.before, [{"case_id": "case-1", "route": "alpha"}])
|
|
||||||
write_json(
|
|
||||||
self.after,
|
|
||||||
[
|
|
||||||
{"case_id": "case-1", "route": "alpha"},
|
|
||||||
{"case_id": "case-new", "route": "delta"},
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
result, summary, _ = self.run_audit()
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual(summary["missing_before_cases"], ["case-new"])
|
|
||||||
self.assertEqual(summary["missing_after_cases"], [])
|
|
||||||
|
|
||||||
def test_case_missing_in_after_is_reported(self) -> None:
|
|
||||||
write_json(
|
|
||||||
self.before,
|
|
||||||
[
|
|
||||||
{"case_id": "case-1", "route": "alpha"},
|
|
||||||
{"case_id": "case-removed", "route": "delta"},
|
|
||||||
],
|
|
||||||
)
|
|
||||||
write_json(self.after, [{"case_id": "case-1", "route": "alpha"}])
|
|
||||||
|
|
||||||
result, summary, _ = self.run_audit()
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual(summary["missing_before_cases"], [])
|
|
||||||
self.assertEqual(summary["missing_after_cases"], ["case-removed"])
|
|
||||||
|
|
||||||
def test_changed_no_call_status_is_visible_in_diff_table(self) -> None:
|
|
||||||
write_json(self.before, [{"case_id": "case-1", "route": "no_call"}])
|
|
||||||
write_json(self.after, [{"case_id": "case-1", "route": "alpha"}])
|
|
||||||
|
|
||||||
result, summary, _ = self.run_audit()
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual(summary["changed_cases"], ["case-1"])
|
|
||||||
self.assertTrue(summary["diff_table"][0]["no_call_changed"])
|
|
||||||
|
|
||||||
def test_new_failures_and_resolved_failures_are_classified(self) -> None:
|
|
||||||
write_json(
|
|
||||||
self.before,
|
|
||||||
[
|
|
||||||
{"case_id": "case-new-fail", "route": "alpha", "expected_route": "alpha"},
|
|
||||||
{"case_id": "case-resolved", "route": "wrong", "expected_route": "beta"},
|
|
||||||
],
|
|
||||||
)
|
|
||||||
write_json(
|
|
||||||
self.after,
|
|
||||||
[
|
|
||||||
{"case_id": "case-new-fail", "route": "wrong", "expected_route": "alpha"},
|
|
||||||
{"case_id": "case-resolved", "route": "beta", "expected_route": "beta"},
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
result, summary, _ = self.run_audit()
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual(summary["new_failures"], ["case-new-fail"])
|
|
||||||
self.assertEqual(summary["resolved_failures"], ["case-resolved"])
|
|
||||||
|
|
||||||
def test_utf8_paths_and_chinese_filenames_are_preserved(self) -> None:
|
|
||||||
before_dir = self.root / "之前"
|
|
||||||
after_dir = self.root / "之后"
|
|
||||||
self.before = before_dir / "路由结果.json"
|
|
||||||
self.after = after_dir / "路由结果.json"
|
|
||||||
write_json(self.before, [{"case_id": "案例一", "route": "旧路由"}])
|
|
||||||
write_json(self.after, [{"case_id": "案例一", "route": "新路由"}])
|
|
||||||
|
|
||||||
result, summary, markdown = self.run_audit()
|
|
||||||
|
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
|
||||||
self.assertEqual(summary["changed_cases"], ["案例一"])
|
|
||||||
self.assertEqual(summary["diff_table"][0]["before_route"], "旧路由")
|
|
||||||
self.assertEqual(summary["diff_table"][0]["after_route"], "新路由")
|
|
||||||
self.assertIn("案例一", markdown)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
unittest.main()
|
|
||||||
Loading…
Reference in New Issue