feat: add regression validation gate runner skill

This commit is contained in:
wantsong 2026-06-19 03:06:15 +08:00
parent a2e8f1bd7a
commit 625ab2e516
8 changed files with 966 additions and 0 deletions

View File

@ -24,6 +24,7 @@ PowerShell:
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill bundle-zip powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill bundle-zip
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill lifecycle-status-guard-scan powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill lifecycle-status-guard-scan
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill repair-markdown-citations powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill repair-markdown-citations
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill regression-validation-gate-runner
``` ```
Use `-Force` to back up and replace an existing installed copy: Use `-Force` to back up and replace an existing installed copy:
@ -32,6 +33,7 @@ Use `-Force` to back up and replace an existing installed copy:
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill bundle-zip -Force powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill bundle-zip -Force
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill lifecycle-status-guard-scan -Force powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill lifecycle-status-guard-scan -Force
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill repair-markdown-citations -Force powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill repair-markdown-citations -Force
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill regression-validation-gate-runner -Force
``` ```
`-All` installs only registry rows with an installable status and `default` install policy. Optional Skills must be named explicitly. `-All` installs only registry rows with an installable status and `default` install policy. Optional Skills must be named explicitly.

View File

@ -17,6 +17,7 @@ Existing CCPE content is not migrated here.
| `fix-title` | Shift Markdown ATX heading depth for pasted LLM replies before insertion under parent sections. | `C:\Users\wangq\Documents\Codex\knowledge-vault\skills\fix-title` | `installed` | `default` | `C:\Users\wangq\.agents\skills\fix-title` | none | | `fix-title` | Shift Markdown ATX heading depth for pasted LLM replies before insertion under parent sections. | `C:\Users\wangq\Documents\Codex\knowledge-vault\skills\fix-title` | `installed` | `default` | `C:\Users\wangq\.agents\skills\fix-title` | none |
| `lifecycle-status-guard-scan` | Scan configured Markdown, JSON, YAML, and text files for lifecycle/status overclaim candidates and missing local evidence markers. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\lifecycle-status-guard-scan` | `installed` | `default` | `C:\Users\wangq\.agents\skills\lifecycle-status-guard-scan` | none | | `lifecycle-status-guard-scan` | Scan configured Markdown, JSON, YAML, and text files for lifecycle/status overclaim candidates and missing local evidence markers. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\lifecycle-status-guard-scan` | `installed` | `default` | `C:\Users\wangq\.agents\skills\lifecycle-status-guard-scan` | none |
| `repair-markdown-citations` | Repair ChatGPT/Deep Research Markdown citation tokens into standard footnotes and a deduplicated reference section using exact turn-ID metadata mapping. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\repair-markdown-citations` | `installed` | `default` | `C:\Users\wangq\.agents\skills\repair-markdown-citations` | none | | `repair-markdown-citations` | Repair ChatGPT/Deep Research Markdown citation tokens into standard footnotes and a deduplicated reference section using exact turn-ID metadata mapping. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\repair-markdown-citations` | `installed` | `default` | `C:\Users\wangq\.agents\skills\repair-markdown-citations` | none |
| `regression-validation-gate-runner` | Dry-run or execute project-declared regression and validation gates from a manifest while capturing logs, exit codes, durations, skipped gates, and changed-file notes. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\regression-validation-gate-runner` | `installed` | `default` | `C:\Users\wangq\.agents\skills\regression-validation-gate-runner` | none |
| `voice-generation` | Generate spoken audio from Markdown scripts using the local `voice-gen` CLI and MiniMax `mmx`, including custom voice clone registration and voice registry management. | `D:\AI\ClaudeCode\voice-ge` | `installed` | `conditional` | `C:\Users\wangq\.agents\skills\voice-generation` | none | | `voice-generation` | Generate spoken audio from Markdown scripts using the local `voice-gen` CLI and MiniMax `mmx`, including custom voice clone registration and voice registry management. | `D:\AI\ClaudeCode\voice-ge` | `installed` | `conditional` | `C:\Users\wangq\.agents\skills\voice-generation` | none |
## Status Values ## Status Values

View File

@ -0,0 +1,42 @@
# Regression Validation Gate Runner
Source Skill for running or dry-running declared regression and validation gates while capturing durable logs and machine-readable reports.
The canonical entry point is `SKILL.md`; the deterministic runner is in `scripts/regression_validation_gate_runner.py`.
## Original Source
```text
Original source: C:\Users\wangq\Documents\Codex\ccpe-system\requirements\skills-vault\2026-06-19-regression-validation-gate-runner.md
Migration date: 2026-06-19
Migration status: first public Skill implementation
Behavior preserved: manifest-declared command execution only; no review judgment or lifecycle approval
Known gaps: no project-specific gate presets beyond the example fixture
```
## Layout
```text
SKILL.md
README.md
agents/openai.yaml
fixtures/gate-manifest.example.yaml
scripts/regression_validation_gate_runner.py
tests/test_regression_validation_gate_runner.py
```
## Usage
```powershell
conda run -n skills-vault python .\scripts\regression_validation_gate_runner.py `
--project-root C:\path\project `
--gate-manifest C:\path\gate-manifest.yaml `
--output-dir C:\path\helper-outputs `
--mode dry_run
```
## Tests
```powershell
conda run -n skills-vault python -B -m unittest discover -s skills/regression-validation-gate-runner/tests -v
```

View File

@ -0,0 +1,98 @@
---
name: regression-validation-gate-runner
description: Use when Codex needs to dry-run or execute project-declared regression, validation, test, lint, build, schema, export, or review gates from a JSON, YAML, or Markdown manifest and capture logs before review, release, handoff, CCRA/local review, or lifecycle-sensitive work.
---
# Regression Validation Gate Runner
## Purpose
Run or verify explicitly declared validation gates and write a durable evidence package. Use the bundled script instead of ad hoc terminal copying when a reviewer needs command declarations, exit codes, logs, durations, skipped gates, and changed-file notes.
Passing gates are engineering evidence only. They do not imply product acceptance, lifecycle approval, Owner approval, or CCRA approval.
## Command
```powershell
conda run -n skills-vault python .\scripts\regression_validation_gate_runner.py `
--project-root C:\path\project `
--gate-manifest C:\path\gate-manifest.yaml `
--output-dir C:\path\helper-outputs `
--mode dry_run
```
Use `--mode run` only when the user has asked to execute the manifest commands. Add `--include-optional` only when optional gates should run.
When installed under `C:\Users\wangq\.agents\skills\regression-validation-gate-runner`, run from that Skill directory or use the installed script path.
## Manifest
The manifest may be JSON, YAML, YML, or Markdown containing one fenced `yaml`, `yml`, or `json` block.
```yaml
gates:
- gate_id: unit-tests
command:
- conda
- run
- -n
- skills-vault
- python
- -B
- -m
- unittest
- discover
- -s
- skills/example/tests
- -v
working_directory: .
expected_exit_code: 0
log_file: logs/unit-tests.log
timeout_seconds: 120
required_before_review: true
```
`command` may be a string or an argv list. Lists avoid shell quoting surprises on Windows.
## Modes
- `dry_run`: parse and validate the manifest, then report declared and skipped gates without executing commands.
- `run`: execute required gates. Optional gates are skipped unless `--include-optional` is passed.
## Outputs
The script writes under `output_dir`:
```text
gate-run-report.md
gate-run-report.json
logs/<gate-id>.log
```
Reports include:
- commands declared, run, and skipped
- exit codes and pass/fail/timeout status
- duration and log paths
- environment notes
- project files changed during each gate
- machine-readable summary
Exit code is `0` for `PASS` or `DRY_RUN`, `1` for failed or timed-out required gates, and `2` for manifest/input errors.
## Safety
- Run only commands explicitly listed in the manifest.
- Keep `working_directory` under `project_root`.
- Write logs and reports only under `output_dir`.
- Do not install dependencies, invent commands, edit project files directly, or repair failures.
- Record changed project files if a command creates, modifies, or deletes files; do not hide side effects.
## Validation
After changing runner behavior, run:
```powershell
conda run -n skills-vault python -B -m unittest discover -s skills/regression-validation-gate-runner/tests -v
conda run -n skills-vault python -B C:\Users\wangq\.codex\skills\.system\skill-creator\scripts\quick_validate.py skills/regression-validation-gate-runner
```

View File

@ -0,0 +1,4 @@
interface:
display_name: "Regression Validation Gate Runner"
short_description: "Run declared validation gates with logs"
default_prompt: "Use $regression-validation-gate-runner to dry-run or execute declared project gates and capture review-ready logs."

View File

@ -0,0 +1,27 @@
gates:
- gate_id: unit-tests
command:
- conda
- run
- -n
- skills-vault
- python
- -B
- -m
- unittest
- discover
- -s
- skills/example/tests
- -v
working_directory: .
expected_exit_code: 0
log_file: logs/unit-tests.log
timeout_seconds: 120
required_before_review: true
- gate_id: packaging-check
command: "python -m build --sdist --wheel"
working_directory: .
expected_exit_code: 0
log_file: logs/packaging-check.log
timeout_seconds: 300
required_before_review: false

View File

@ -0,0 +1,486 @@
from __future__ import annotations
import argparse
import json
import os
import pathlib
import platform
import re
import subprocess
import sys
import time
from collections.abc import Iterable, Mapping
from dataclasses import dataclass
from typing import Any
import yaml
REPORT_JSON = "gate-run-report.json"
REPORT_MD = "gate-run-report.md"
class ManifestError(ValueError):
pass
@dataclass
class Gate:
gate_id: str
command: str | list[str]
working_directory: pathlib.Path
expected_exit_code: int
log_file: pathlib.Path
timeout_seconds: float
required_before_review: bool
def load_raw_manifest(path: pathlib.Path) -> Any:
text = path.read_text(encoding="utf-8")
suffix = path.suffix.lower()
if suffix == ".json":
return json.loads(text)
if suffix in {".yaml", ".yml"}:
return yaml.safe_load(text)
if suffix in {".md", ".markdown"}:
fenced = re.search(r"```(yaml|yml|json)\s*\n(.*?)\n```", text, flags=re.DOTALL | re.I)
if not fenced:
raise ManifestError("Markdown manifest must contain a fenced yaml, yml, or json block.")
language = fenced.group(1).casefold()
body = fenced.group(2)
return json.loads(body) if language == "json" else yaml.safe_load(body)
raise ManifestError("Gate manifest must be JSON, YAML, YML, or Markdown.")
def manifest_gates(raw: Any) -> list[Mapping[str, Any]]:
if isinstance(raw, Mapping):
gates = raw.get("gates")
else:
gates = raw
if not isinstance(gates, list):
raise ManifestError("Gate manifest must define a gates list.")
normalized: list[Mapping[str, Any]] = []
for index, item in enumerate(gates):
if not isinstance(item, Mapping):
raise ManifestError(f"Gate at index {index} must be an object.")
normalized.append(item)
return normalized
def bool_value(value: Any, default: bool) -> bool:
if value is None:
return default
if isinstance(value, bool):
return value
if isinstance(value, str):
return value.strip().casefold() in {"1", "true", "yes", "y"}
return bool(value)
def validate_gate_id(value: Any) -> str:
if not isinstance(value, str) or not value.strip():
raise ManifestError("Every gate must define a non-empty gate_id.")
gate_id = value.strip()
if any(separator in gate_id for separator in ("/", "\\", os.sep)):
raise ManifestError(f"gate_id must not contain path separators: {gate_id}")
return gate_id
def validate_command(value: Any, gate_id: str) -> str | list[str]:
if isinstance(value, str) and value.strip():
return value
if isinstance(value, list) and value and all(isinstance(item, str) and item for item in value):
return value
raise ManifestError(f"Gate {gate_id} must define command as a non-empty string or string list.")
def resolve_under_root(base: pathlib.Path, raw_path: Any, default: str, label: str) -> pathlib.Path:
candidate_text = default if raw_path in {None, ""} else str(raw_path)
candidate = pathlib.Path(candidate_text)
if not candidate.is_absolute():
candidate = base / candidate
resolved = candidate.resolve()
try:
resolved.relative_to(base)
except ValueError as exc:
raise ManifestError(f"{label} must resolve under project_root: {candidate_text}") from exc
return resolved
def resolve_log_file(output_dir: pathlib.Path, raw_path: Any, gate_id: str) -> pathlib.Path:
candidate_text = f"logs/{gate_id}.log" if raw_path in {None, ""} else str(raw_path)
candidate = pathlib.Path(candidate_text)
if candidate.is_absolute():
resolved = candidate.resolve()
else:
resolved = (output_dir / candidate).resolve()
try:
resolved.relative_to(output_dir)
except ValueError as exc:
raise ManifestError(f"log_file must resolve under output_dir for gate {gate_id}: {candidate_text}") from exc
return resolved
def parse_gates(
raw_manifest: Any,
project_root: pathlib.Path,
output_dir: pathlib.Path,
) -> tuple[list[Gate], list[str], list[str]]:
gates: list[Gate] = []
errors: list[str] = []
declared: list[str] = []
for index, raw_gate in enumerate(manifest_gates(raw_manifest)):
try:
gate_id = validate_gate_id(raw_gate.get("gate_id"))
declared.append(gate_id)
command = validate_command(raw_gate.get("command"), gate_id)
working_directory = resolve_under_root(
project_root,
raw_gate.get("working_directory"),
".",
f"working_directory for gate {gate_id}",
)
expected_exit_code = int(raw_gate.get("expected_exit_code", 0))
timeout_seconds = float(raw_gate.get("timeout_seconds", 300))
if timeout_seconds <= 0:
raise ManifestError(f"timeout_seconds must be positive for gate {gate_id}.")
log_file = resolve_log_file(output_dir, raw_gate.get("log_file"), gate_id)
required = bool_value(raw_gate.get("required_before_review"), True)
gates.append(
Gate(
gate_id=gate_id,
command=command,
working_directory=working_directory,
expected_exit_code=expected_exit_code,
log_file=log_file,
timeout_seconds=timeout_seconds,
required_before_review=required,
)
)
except ManifestError as exc:
errors.append(f"gate[{index}]: {exc}")
return gates, errors, declared
def empty_report(project_root: pathlib.Path, manifest_path: pathlib.Path, mode: str) -> dict[str, Any]:
return {
"project_root": str(project_root),
"gate_manifest_path": str(manifest_path),
"mode": mode,
"commands_declared": [],
"commands_run": [],
"commands_skipped": [],
"exit_codes": {},
"pass_fail_status": "PENDING",
"duration": 0.0,
"log_paths": {},
"environment_notes": [
"Commands are declared by the manifest.",
"Passing gates are engineering evidence only; they do not imply product acceptance or lifecycle approval.",
],
"gate_results": [],
"manifest_errors": [],
"changed_files": [],
"machine_readable_summary": {},
}
def should_exclude(path: pathlib.Path, output_dir: pathlib.Path, project_root: pathlib.Path) -> bool:
try:
path.relative_to(output_dir)
return True
except ValueError:
pass
try:
path.relative_to(project_root / ".git")
return True
except ValueError:
return False
def snapshot_files(project_root: pathlib.Path, output_dir: pathlib.Path) -> dict[str, tuple[int, int]]:
snapshot: dict[str, tuple[int, int]] = {}
for path in project_root.rglob("*"):
if not path.is_file() or should_exclude(path, output_dir, project_root):
continue
stat = path.stat()
snapshot[path.relative_to(project_root).as_posix()] = (stat.st_size, stat.st_mtime_ns)
return snapshot
def diff_snapshots(before: dict[str, tuple[int, int]], after: dict[str, tuple[int, int]]) -> list[str]:
changed = {
path
for path in set(before) | set(after)
if before.get(path) != after.get(path)
}
return sorted(changed)
def command_display(command: str | list[str]) -> str:
if isinstance(command, list):
return " ".join(command)
return command
def command_for_subprocess(command: str | list[str]) -> tuple[str | list[str], bool]:
if isinstance(command, list):
return command, False
return command, True
def safe_text(value: Any) -> str:
if value is None:
return ""
if isinstance(value, bytes):
return value.decode("utf-8", errors="replace")
return str(value)
def run_gate(
gate: Gate,
project_root: pathlib.Path,
output_dir: pathlib.Path,
) -> dict[str, Any]:
start = time.monotonic()
before = snapshot_files(project_root, output_dir)
command, shell = command_for_subprocess(gate.command)
status = "PENDING"
exit_code: int | None = None
stdout = ""
stderr = ""
timeout = False
try:
completed = subprocess.run(
command,
cwd=str(gate.working_directory),
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=gate.timeout_seconds,
shell=shell,
check=False,
)
exit_code = completed.returncode
stdout = completed.stdout
stderr = completed.stderr
status = "PASS" if exit_code == gate.expected_exit_code else "FAIL"
except subprocess.TimeoutExpired as exc:
timeout = True
stdout = safe_text(exc.stdout or exc.output)
stderr = safe_text(exc.stderr)
status = "TIMEOUT"
duration = time.monotonic() - start
after = snapshot_files(project_root, output_dir)
changed_files = diff_snapshots(before, after)
gate.log_file.parent.mkdir(parents=True, exist_ok=True)
log_lines = [
f"gate_id: {gate.gate_id}",
f"working_directory: {gate.working_directory}",
f"command: {command_display(gate.command)}",
f"expected_exit_code: {gate.expected_exit_code}",
f"exit_code: {exit_code}",
f"status: {status}",
f"duration_seconds: {duration:.3f}",
"",
"## stdout",
stdout,
"",
"## stderr",
stderr,
]
if timeout:
log_lines.extend(["", f"Command timed out after {gate.timeout_seconds} seconds."])
gate.log_file.write_text("\n".join(log_lines), encoding="utf-8")
return {
"gate_id": gate.gate_id,
"command": command_display(gate.command),
"working_directory": str(gate.working_directory),
"expected_exit_code": gate.expected_exit_code,
"exit_code": exit_code,
"status": status,
"duration": duration,
"log_path": gate.log_file.relative_to(output_dir).as_posix(),
"required_before_review": gate.required_before_review,
"timeout": timeout,
"changed_files": changed_files,
}
def finalize_report(report: dict[str, Any]) -> None:
if report["manifest_errors"]:
status = "ERROR"
elif report["mode"] == "dry_run":
status = "DRY_RUN"
elif any(result["status"] in {"FAIL", "TIMEOUT"} for result in report["gate_results"]):
status = "FAIL"
else:
status = "PASS"
report["pass_fail_status"] = status
report["duration"] = round(float(report["duration"]), 3)
report["changed_files"] = sorted(
{
changed
for result in report["gate_results"]
for changed in result.get("changed_files", [])
}
)
report["machine_readable_summary"] = {
"status": status,
"commands_declared_count": len(report["commands_declared"]),
"commands_run_count": len(report["commands_run"]),
"commands_skipped_count": len(report["commands_skipped"]),
"manifest_error_count": len(report["manifest_errors"]),
"failed_count": sum(1 for result in report["gate_results"] if result["status"] == "FAIL"),
"timeout_count": sum(1 for result in report["gate_results"] if result["status"] == "TIMEOUT"),
"changed_file_count": len(report["changed_files"]),
}
def write_json_report(report: dict[str, Any], output_dir: pathlib.Path) -> None:
(output_dir / REPORT_JSON).write_text(
json.dumps(report, ensure_ascii=False, indent=2),
encoding="utf-8",
)
def write_markdown_report(report: dict[str, Any], output_dir: pathlib.Path) -> None:
summary = report["machine_readable_summary"]
lines = [
"# Gate Run Report",
"",
"## Summary",
"",
f"- Status: `{summary['status']}`",
f"- Commands declared: {summary['commands_declared_count']}",
f"- Commands run: {summary['commands_run_count']}",
f"- Commands skipped: {summary['commands_skipped_count']}",
f"- Manifest errors: {summary['manifest_error_count']}",
f"- Changed files: {summary['changed_file_count']}",
"",
"## Gate Results",
"",
]
if report["gate_results"]:
for result in report["gate_results"]:
lines.append(
f"- `{result['gate_id']}`: `{result['status']}` exit `{result['exit_code']}` log `{result['log_path']}`"
)
else:
lines.append("- None")
lines.extend(["", "## Skipped Gates", ""])
if report["commands_skipped"]:
for skipped in report["commands_skipped"]:
lines.append(f"- `{skipped['gate_id']}`: {skipped['reason']}")
else:
lines.append("- None")
lines.extend(["", "## Manifest Errors", ""])
if report["manifest_errors"]:
for error in report["manifest_errors"]:
lines.append(f"- {error}")
else:
lines.append("- None")
lines.extend(["", "## Changed Files", ""])
if report["changed_files"]:
for path in report["changed_files"]:
lines.append(f"- `{path}`")
else:
lines.append("- None")
(output_dir / REPORT_MD).write_text("\n".join(lines) + "\n", encoding="utf-8")
def write_reports(report: dict[str, Any], output_dir: pathlib.Path) -> None:
output_dir.mkdir(parents=True, exist_ok=True)
write_json_report(report, output_dir)
write_markdown_report(report, output_dir)
def run(
project_root: pathlib.Path,
manifest_path: pathlib.Path,
output_dir: pathlib.Path,
mode: str,
include_optional: bool = False,
) -> int:
project_root = project_root.resolve()
manifest_path = manifest_path.resolve()
output_dir = output_dir.resolve()
output_dir.mkdir(parents=True, exist_ok=True)
report = empty_report(project_root, manifest_path, mode)
report["environment_notes"].extend(
[
f"python: {sys.version.split()[0]}",
f"platform: {platform.platform()}",
]
)
if not project_root.is_dir():
report["manifest_errors"].append(f"project_root is not a directory: {project_root}")
finalize_report(report)
write_reports(report, output_dir)
return 2
if not manifest_path.is_file():
report["manifest_errors"].append(f"gate_manifest_path does not exist: {manifest_path}")
finalize_report(report)
write_reports(report, output_dir)
return 2
try:
raw_manifest = load_raw_manifest(manifest_path)
gates, errors, declared = parse_gates(raw_manifest, project_root, output_dir)
report["commands_declared"] = declared
report["manifest_errors"].extend(errors)
except Exception as exc:
report["manifest_errors"].append(str(exc))
finalize_report(report)
write_reports(report, output_dir)
return 2
if report["manifest_errors"]:
finalize_report(report)
write_reports(report, output_dir)
return 2
run_start = time.monotonic()
for gate in gates:
if mode == "dry_run":
report["commands_skipped"].append({"gate_id": gate.gate_id, "reason": "dry_run"})
continue
if not gate.required_before_review and not include_optional:
report["commands_skipped"].append({"gate_id": gate.gate_id, "reason": "optional"})
continue
result = run_gate(gate, project_root, output_dir)
report["gate_results"].append(result)
report["commands_run"].append(gate.gate_id)
report["exit_codes"][gate.gate_id] = result["exit_code"]
report["log_paths"][gate.gate_id] = result["log_path"]
report["duration"] = time.monotonic() - run_start
finalize_report(report)
write_reports(report, output_dir)
return 0 if report["pass_fail_status"] in {"PASS", "DRY_RUN"} else 1
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="Run or dry-run declared regression validation gates.")
parser.add_argument("--project-root", required=True, type=pathlib.Path)
parser.add_argument("--gate-manifest", required=True, type=pathlib.Path)
parser.add_argument("--output-dir", required=True, type=pathlib.Path)
parser.add_argument("--mode", choices=["dry_run", "run"], required=True)
parser.add_argument("--include-optional", action="store_true")
args = parser.parse_args(argv)
return run(
project_root=args.project_root,
manifest_path=args.gate_manifest,
output_dir=args.output_dir,
mode=args.mode,
include_optional=args.include_optional,
)
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,306 @@
from __future__ import annotations
import json
import pathlib
import shutil
import subprocess
import sys
import tempfile
import textwrap
import unittest
SKILL_ROOT = pathlib.Path(__file__).resolve().parents[1]
REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]
SCRIPT = SKILL_ROOT / "scripts" / "regression_validation_gate_runner.py"
TMP_ROOT = REPO_ROOT / "tmp" / "regression-validation-gate-runner-tests"
def quote(value: str) -> str:
return json.dumps(value)
def py_command(source: str) -> list[str]:
return [sys.executable, "-c", source]
def run_cli(*args: str) -> subprocess.CompletedProcess[str]:
return subprocess.run(
[sys.executable, str(SCRIPT), *args],
text=True,
capture_output=True,
check=False,
)
def write_text(path: pathlib.Path, text: str) -> pathlib.Path:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, encoding="utf-8")
return path
def write_json(path: pathlib.Path, data: object) -> pathlib.Path:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
return path
def load_report(output_dir: pathlib.Path) -> dict[str, object]:
return json.loads((output_dir / "gate-run-report.json").read_text(encoding="utf-8"))
class RegressionValidationGateRunnerTest(unittest.TestCase):
def setUp(self) -> None:
TMP_ROOT.mkdir(parents=True, exist_ok=True)
self.tmp_dir = tempfile.TemporaryDirectory(dir=TMP_ROOT)
self.root = pathlib.Path(self.tmp_dir.name) / "project"
self.output_dir = pathlib.Path(self.tmp_dir.name) / "outputs"
self.manifest = pathlib.Path(self.tmp_dir.name) / "gates.json"
self.root.mkdir()
def tearDown(self) -> None:
self.tmp_dir.cleanup()
@classmethod
def tearDownClass(cls) -> None:
if TMP_ROOT.exists():
shutil.rmtree(TMP_ROOT)
def run_gate_runner(self, mode: str = "run") -> tuple[subprocess.CompletedProcess[str], dict[str, object]]:
result = run_cli(
"--project-root",
str(self.root),
"--gate-manifest",
str(self.manifest),
"--output-dir",
str(self.output_dir),
"--mode",
mode,
)
return result, load_report(self.output_dir)
def test_dry_run_parses_json_manifest_without_running_commands(self) -> None:
marker = self.root / "should-not-exist.txt"
write_json(
self.manifest,
{
"gates": [
{
"gate_id": "unit",
"command": py_command(f"open({quote(str(marker))}, 'w').write('ran')"),
"working_directory": ".",
"expected_exit_code": 0,
"timeout_seconds": 5,
"required_before_review": True,
}
]
},
)
result, report = self.run_gate_runner(mode="dry_run")
self.assertEqual(result.returncode, 0, result.stderr)
self.assertFalse(marker.exists())
self.assertEqual(report["commands_declared"], ["unit"])
self.assertEqual(report["commands_run"], [])
self.assertEqual(report["commands_skipped"][0]["reason"], "dry_run")
self.assertEqual(report["machine_readable_summary"]["status"], "DRY_RUN")
def test_successful_command_capture_writes_log_and_passes(self) -> None:
write_json(
self.manifest,
{
"gates": [
{
"gate_id": "unit",
"command": py_command("print('hello gate')"),
"working_directory": ".",
"expected_exit_code": 0,
"log_file": "logs/unit.log",
"timeout_seconds": 5,
"required_before_review": True,
}
]
},
)
result, report = self.run_gate_runner()
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(report["pass_fail_status"], "PASS")
self.assertEqual(report["exit_codes"]["unit"], 0)
log_path = self.output_dir / "logs" / "unit.log"
self.assertTrue(log_path.exists())
self.assertIn("hello gate", log_path.read_text(encoding="utf-8"))
self.assertEqual(report["commands_run"], ["unit"])
def test_failing_command_capture_fails_required_gate(self) -> None:
write_json(
self.manifest,
{
"gates": [
{
"gate_id": "regression",
"command": py_command("import sys; print('bad'); sys.exit(3)"),
"expected_exit_code": 0,
"timeout_seconds": 5,
"required_before_review": True,
}
]
},
)
result, report = self.run_gate_runner()
self.assertEqual(result.returncode, 1, result.stderr)
self.assertEqual(report["pass_fail_status"], "FAIL")
self.assertEqual(report["exit_codes"]["regression"], 3)
gate = report["gate_results"][0]
self.assertEqual(gate["status"], "FAIL")
self.assertIn("bad", (self.output_dir / "logs" / "regression.log").read_text(encoding="utf-8"))
def test_optional_gate_is_skipped_by_default(self) -> None:
marker = self.root / "optional-ran.txt"
write_json(
self.manifest,
{
"gates": [
{
"gate_id": "optional-packaging",
"command": py_command(f"open({quote(str(marker))}, 'w').write('ran')"),
"expected_exit_code": 0,
"timeout_seconds": 5,
"required_before_review": False,
}
]
},
)
result, report = self.run_gate_runner()
self.assertEqual(result.returncode, 0, result.stderr)
self.assertFalse(marker.exists())
self.assertEqual(report["commands_run"], [])
self.assertEqual(report["commands_skipped"][0]["gate_id"], "optional-packaging")
self.assertEqual(report["commands_skipped"][0]["reason"], "optional")
def test_missing_command_field_is_manifest_error(self) -> None:
write_json(
self.manifest,
{"gates": [{"gate_id": "broken", "expected_exit_code": 0, "required_before_review": True}]},
)
result, report = self.run_gate_runner()
self.assertEqual(result.returncode, 2)
self.assertEqual(report["pass_fail_status"], "ERROR")
self.assertEqual(report["machine_readable_summary"]["manifest_error_count"], 1)
self.assertIn("command", report["manifest_errors"][0])
def test_timeout_handling_marks_required_gate_failed(self) -> None:
write_json(
self.manifest,
{
"gates": [
{
"gate_id": "slow",
"command": py_command("import time; time.sleep(3)"),
"expected_exit_code": 0,
"timeout_seconds": 1,
"required_before_review": True,
}
]
},
)
result, report = self.run_gate_runner()
self.assertEqual(result.returncode, 1, result.stderr)
gate = report["gate_results"][0]
self.assertEqual(gate["status"], "TIMEOUT")
self.assertEqual(report["pass_fail_status"], "FAIL")
self.assertIn("timed out", (self.output_dir / "logs" / "slow.log").read_text(encoding="utf-8"))
def test_markdown_manifest_with_yaml_fence_is_supported(self) -> None:
self.manifest = pathlib.Path(self.tmp_dir.name) / "gates.md"
write_text(
self.manifest,
textwrap.dedent(
f"""\
# Gates
```yaml
gates:
- gate_id: markdown-gate
command:
- {sys.executable}
- -c
- "print('from markdown')"
expected_exit_code: 0
timeout_seconds: 5
required_before_review: true
```
"""
),
)
result, report = self.run_gate_runner()
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(report["commands_declared"], ["markdown-gate"])
self.assertEqual(report["commands_run"], ["markdown-gate"])
def test_utf8_paths_and_chinese_filenames_are_logged(self) -> None:
project = pathlib.Path(self.tmp_dir.name) / "项目"
project.mkdir()
self.root = project
write_json(
self.manifest,
{
"gates": [
{
"gate_id": "中文-gate",
"command": py_command("print('中文输出')"),
"working_directory": ".",
"expected_exit_code": 0,
"timeout_seconds": 5,
"required_before_review": True,
}
]
},
)
result, report = self.run_gate_runner()
self.assertEqual(result.returncode, 0, result.stderr)
log_path = self.output_dir / "logs" / "中文-gate.log"
self.assertTrue(log_path.exists())
self.assertIn("中文输出", log_path.read_text(encoding="utf-8"))
self.assertIn("中文-gate", report["log_paths"])
def test_project_file_changes_are_reported(self) -> None:
changed = self.root / "generated.txt"
write_json(
self.manifest,
{
"gates": [
{
"gate_id": "side-effect",
"command": py_command(f"open({quote(str(changed))}, 'w', encoding='utf-8').write('new')"),
"expected_exit_code": 0,
"timeout_seconds": 5,
"required_before_review": True,
}
]
},
)
result, report = self.run_gate_runner()
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(report["changed_files"], ["generated.txt"])
self.assertEqual(report["gate_results"][0]["changed_files"], ["generated.txt"])
if __name__ == "__main__":
unittest.main()