feat: add regression validation gate runner skill
This commit is contained in:
parent
a2e8f1bd7a
commit
625ab2e516
|
|
@ -24,6 +24,7 @@ PowerShell:
|
||||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill bundle-zip
|
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill bundle-zip
|
||||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill lifecycle-status-guard-scan
|
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill lifecycle-status-guard-scan
|
||||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill repair-markdown-citations
|
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill repair-markdown-citations
|
||||||
|
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill regression-validation-gate-runner
|
||||||
```
|
```
|
||||||
|
|
||||||
Use `-Force` to back up and replace an existing installed copy:
|
Use `-Force` to back up and replace an existing installed copy:
|
||||||
|
|
@ -32,6 +33,7 @@ Use `-Force` to back up and replace an existing installed copy:
|
||||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill bundle-zip -Force
|
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill bundle-zip -Force
|
||||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill lifecycle-status-guard-scan -Force
|
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill lifecycle-status-guard-scan -Force
|
||||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill repair-markdown-citations -Force
|
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill repair-markdown-citations -Force
|
||||||
|
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill regression-validation-gate-runner -Force
|
||||||
```
|
```
|
||||||
|
|
||||||
`-All` installs only registry rows with an installable status and `default` install policy. Optional Skills must be named explicitly.
|
`-All` installs only registry rows with an installable status and `default` install policy. Optional Skills must be named explicitly.
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,7 @@ Existing CCPE content is not migrated here.
|
||||||
| `fix-title` | Shift Markdown ATX heading depth for pasted LLM replies before insertion under parent sections. | `C:\Users\wangq\Documents\Codex\knowledge-vault\skills\fix-title` | `installed` | `default` | `C:\Users\wangq\.agents\skills\fix-title` | none |
|
| `fix-title` | Shift Markdown ATX heading depth for pasted LLM replies before insertion under parent sections. | `C:\Users\wangq\Documents\Codex\knowledge-vault\skills\fix-title` | `installed` | `default` | `C:\Users\wangq\.agents\skills\fix-title` | none |
|
||||||
| `lifecycle-status-guard-scan` | Scan configured Markdown, JSON, YAML, and text files for lifecycle/status overclaim candidates and missing local evidence markers. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\lifecycle-status-guard-scan` | `installed` | `default` | `C:\Users\wangq\.agents\skills\lifecycle-status-guard-scan` | none |
|
| `lifecycle-status-guard-scan` | Scan configured Markdown, JSON, YAML, and text files for lifecycle/status overclaim candidates and missing local evidence markers. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\lifecycle-status-guard-scan` | `installed` | `default` | `C:\Users\wangq\.agents\skills\lifecycle-status-guard-scan` | none |
|
||||||
| `repair-markdown-citations` | Repair ChatGPT/Deep Research Markdown citation tokens into standard footnotes and a deduplicated reference section using exact turn-ID metadata mapping. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\repair-markdown-citations` | `installed` | `default` | `C:\Users\wangq\.agents\skills\repair-markdown-citations` | none |
|
| `repair-markdown-citations` | Repair ChatGPT/Deep Research Markdown citation tokens into standard footnotes and a deduplicated reference section using exact turn-ID metadata mapping. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\repair-markdown-citations` | `installed` | `default` | `C:\Users\wangq\.agents\skills\repair-markdown-citations` | none |
|
||||||
|
| `regression-validation-gate-runner` | Dry-run or execute project-declared regression and validation gates from a manifest while capturing logs, exit codes, durations, skipped gates, and changed-file notes. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\regression-validation-gate-runner` | `installed` | `default` | `C:\Users\wangq\.agents\skills\regression-validation-gate-runner` | none |
|
||||||
| `voice-generation` | Generate spoken audio from Markdown scripts using the local `voice-gen` CLI and MiniMax `mmx`, including custom voice clone registration and voice registry management. | `D:\AI\ClaudeCode\voice-ge` | `installed` | `conditional` | `C:\Users\wangq\.agents\skills\voice-generation` | none |
|
| `voice-generation` | Generate spoken audio from Markdown scripts using the local `voice-gen` CLI and MiniMax `mmx`, including custom voice clone registration and voice registry management. | `D:\AI\ClaudeCode\voice-ge` | `installed` | `conditional` | `C:\Users\wangq\.agents\skills\voice-generation` | none |
|
||||||
|
|
||||||
## Status Values
|
## Status Values
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,42 @@
|
||||||
|
# Regression Validation Gate Runner
|
||||||
|
|
||||||
|
Source Skill for running or dry-running declared regression and validation gates while capturing durable logs and machine-readable reports.
|
||||||
|
|
||||||
|
The canonical entry point is `SKILL.md`; the deterministic runner is in `scripts/regression_validation_gate_runner.py`.
|
||||||
|
|
||||||
|
## Original Source
|
||||||
|
|
||||||
|
```text
|
||||||
|
Original source: C:\Users\wangq\Documents\Codex\ccpe-system\requirements\skills-vault\2026-06-19-regression-validation-gate-runner.md
|
||||||
|
Migration date: 2026-06-19
|
||||||
|
Migration status: first public Skill implementation
|
||||||
|
Behavior preserved: manifest-declared command execution only; no review judgment or lifecycle approval
|
||||||
|
Known gaps: no project-specific gate presets beyond the example fixture
|
||||||
|
```
|
||||||
|
|
||||||
|
## Layout
|
||||||
|
|
||||||
|
```text
|
||||||
|
SKILL.md
|
||||||
|
README.md
|
||||||
|
agents/openai.yaml
|
||||||
|
fixtures/gate-manifest.example.yaml
|
||||||
|
scripts/regression_validation_gate_runner.py
|
||||||
|
tests/test_regression_validation_gate_runner.py
|
||||||
|
```
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
conda run -n skills-vault python .\scripts\regression_validation_gate_runner.py `
|
||||||
|
--project-root C:\path\project `
|
||||||
|
--gate-manifest C:\path\gate-manifest.yaml `
|
||||||
|
--output-dir C:\path\helper-outputs `
|
||||||
|
--mode dry_run
|
||||||
|
```
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
conda run -n skills-vault python -B -m unittest discover -s skills/regression-validation-gate-runner/tests -v
|
||||||
|
```
|
||||||
|
|
@ -0,0 +1,98 @@
|
||||||
|
---
|
||||||
|
name: regression-validation-gate-runner
|
||||||
|
description: Use when Codex needs to dry-run or execute project-declared regression, validation, test, lint, build, schema, export, or review gates from a JSON, YAML, or Markdown manifest and capture logs before review, release, handoff, CCRA/local review, or lifecycle-sensitive work.
|
||||||
|
---
|
||||||
|
|
||||||
|
# Regression Validation Gate Runner
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Run or verify explicitly declared validation gates and write a durable evidence package. Use the bundled script instead of ad hoc terminal copying when a reviewer needs command declarations, exit codes, logs, durations, skipped gates, and changed-file notes.
|
||||||
|
|
||||||
|
Passing gates are engineering evidence only. They do not imply product acceptance, lifecycle approval, Owner approval, or CCRA approval.
|
||||||
|
|
||||||
|
## Command
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
conda run -n skills-vault python .\scripts\regression_validation_gate_runner.py `
|
||||||
|
--project-root C:\path\project `
|
||||||
|
--gate-manifest C:\path\gate-manifest.yaml `
|
||||||
|
--output-dir C:\path\helper-outputs `
|
||||||
|
--mode dry_run
|
||||||
|
```
|
||||||
|
|
||||||
|
Use `--mode run` only when the user has asked to execute the manifest commands. Add `--include-optional` only when optional gates should run.
|
||||||
|
|
||||||
|
When installed under `C:\Users\wangq\.agents\skills\regression-validation-gate-runner`, run from that Skill directory or use the installed script path.
|
||||||
|
|
||||||
|
## Manifest
|
||||||
|
|
||||||
|
The manifest may be JSON, YAML, YML, or Markdown containing one fenced `yaml`, `yml`, or `json` block.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
gates:
|
||||||
|
- gate_id: unit-tests
|
||||||
|
command:
|
||||||
|
- conda
|
||||||
|
- run
|
||||||
|
- -n
|
||||||
|
- skills-vault
|
||||||
|
- python
|
||||||
|
- -B
|
||||||
|
- -m
|
||||||
|
- unittest
|
||||||
|
- discover
|
||||||
|
- -s
|
||||||
|
- skills/example/tests
|
||||||
|
- -v
|
||||||
|
working_directory: .
|
||||||
|
expected_exit_code: 0
|
||||||
|
log_file: logs/unit-tests.log
|
||||||
|
timeout_seconds: 120
|
||||||
|
required_before_review: true
|
||||||
|
```
|
||||||
|
|
||||||
|
`command` may be a string or an argv list. Lists avoid shell quoting surprises on Windows.
|
||||||
|
|
||||||
|
## Modes
|
||||||
|
|
||||||
|
- `dry_run`: parse and validate the manifest, then report declared and skipped gates without executing commands.
|
||||||
|
- `run`: execute required gates. Optional gates are skipped unless `--include-optional` is passed.
|
||||||
|
|
||||||
|
## Outputs
|
||||||
|
|
||||||
|
The script writes under `output_dir`:
|
||||||
|
|
||||||
|
```text
|
||||||
|
gate-run-report.md
|
||||||
|
gate-run-report.json
|
||||||
|
logs/<gate-id>.log
|
||||||
|
```
|
||||||
|
|
||||||
|
Reports include:
|
||||||
|
|
||||||
|
- commands declared, run, and skipped
|
||||||
|
- exit codes and pass/fail/timeout status
|
||||||
|
- duration and log paths
|
||||||
|
- environment notes
|
||||||
|
- project files changed during each gate
|
||||||
|
- machine-readable summary
|
||||||
|
|
||||||
|
Exit code is `0` for `PASS` or `DRY_RUN`, `1` for failed or timed-out required gates, and `2` for manifest/input errors.
|
||||||
|
|
||||||
|
## Safety
|
||||||
|
|
||||||
|
- Run only commands explicitly listed in the manifest.
|
||||||
|
- Keep `working_directory` under `project_root`.
|
||||||
|
- Write logs and reports only under `output_dir`.
|
||||||
|
- Do not install dependencies, invent commands, edit project files directly, or repair failures.
|
||||||
|
- Record changed project files if a command creates, modifies, or deletes files; do not hide side effects.
|
||||||
|
|
||||||
|
## Validation
|
||||||
|
|
||||||
|
After changing runner behavior, run:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
conda run -n skills-vault python -B -m unittest discover -s skills/regression-validation-gate-runner/tests -v
|
||||||
|
conda run -n skills-vault python -B C:\Users\wangq\.codex\skills\.system\skill-creator\scripts\quick_validate.py skills/regression-validation-gate-runner
|
||||||
|
```
|
||||||
|
|
@ -0,0 +1,4 @@
|
||||||
|
interface:
|
||||||
|
display_name: "Regression Validation Gate Runner"
|
||||||
|
short_description: "Run declared validation gates with logs"
|
||||||
|
default_prompt: "Use $regression-validation-gate-runner to dry-run or execute declared project gates and capture review-ready logs."
|
||||||
|
|
@ -0,0 +1,27 @@
|
||||||
|
gates:
|
||||||
|
- gate_id: unit-tests
|
||||||
|
command:
|
||||||
|
- conda
|
||||||
|
- run
|
||||||
|
- -n
|
||||||
|
- skills-vault
|
||||||
|
- python
|
||||||
|
- -B
|
||||||
|
- -m
|
||||||
|
- unittest
|
||||||
|
- discover
|
||||||
|
- -s
|
||||||
|
- skills/example/tests
|
||||||
|
- -v
|
||||||
|
working_directory: .
|
||||||
|
expected_exit_code: 0
|
||||||
|
log_file: logs/unit-tests.log
|
||||||
|
timeout_seconds: 120
|
||||||
|
required_before_review: true
|
||||||
|
- gate_id: packaging-check
|
||||||
|
command: "python -m build --sdist --wheel"
|
||||||
|
working_directory: .
|
||||||
|
expected_exit_code: 0
|
||||||
|
log_file: logs/packaging-check.log
|
||||||
|
timeout_seconds: 300
|
||||||
|
required_before_review: false
|
||||||
|
|
@ -0,0 +1,486 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import platform
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from collections.abc import Iterable, Mapping
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
|
||||||
|
REPORT_JSON = "gate-run-report.json"
|
||||||
|
REPORT_MD = "gate-run-report.md"
|
||||||
|
|
||||||
|
|
||||||
|
class ManifestError(ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Gate:
|
||||||
|
gate_id: str
|
||||||
|
command: str | list[str]
|
||||||
|
working_directory: pathlib.Path
|
||||||
|
expected_exit_code: int
|
||||||
|
log_file: pathlib.Path
|
||||||
|
timeout_seconds: float
|
||||||
|
required_before_review: bool
|
||||||
|
|
||||||
|
|
||||||
|
def load_raw_manifest(path: pathlib.Path) -> Any:
|
||||||
|
text = path.read_text(encoding="utf-8")
|
||||||
|
suffix = path.suffix.lower()
|
||||||
|
if suffix == ".json":
|
||||||
|
return json.loads(text)
|
||||||
|
if suffix in {".yaml", ".yml"}:
|
||||||
|
return yaml.safe_load(text)
|
||||||
|
if suffix in {".md", ".markdown"}:
|
||||||
|
fenced = re.search(r"```(yaml|yml|json)\s*\n(.*?)\n```", text, flags=re.DOTALL | re.I)
|
||||||
|
if not fenced:
|
||||||
|
raise ManifestError("Markdown manifest must contain a fenced yaml, yml, or json block.")
|
||||||
|
language = fenced.group(1).casefold()
|
||||||
|
body = fenced.group(2)
|
||||||
|
return json.loads(body) if language == "json" else yaml.safe_load(body)
|
||||||
|
raise ManifestError("Gate manifest must be JSON, YAML, YML, or Markdown.")
|
||||||
|
|
||||||
|
|
||||||
|
def manifest_gates(raw: Any) -> list[Mapping[str, Any]]:
|
||||||
|
if isinstance(raw, Mapping):
|
||||||
|
gates = raw.get("gates")
|
||||||
|
else:
|
||||||
|
gates = raw
|
||||||
|
if not isinstance(gates, list):
|
||||||
|
raise ManifestError("Gate manifest must define a gates list.")
|
||||||
|
normalized: list[Mapping[str, Any]] = []
|
||||||
|
for index, item in enumerate(gates):
|
||||||
|
if not isinstance(item, Mapping):
|
||||||
|
raise ManifestError(f"Gate at index {index} must be an object.")
|
||||||
|
normalized.append(item)
|
||||||
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
|
def bool_value(value: Any, default: bool) -> bool:
|
||||||
|
if value is None:
|
||||||
|
return default
|
||||||
|
if isinstance(value, bool):
|
||||||
|
return value
|
||||||
|
if isinstance(value, str):
|
||||||
|
return value.strip().casefold() in {"1", "true", "yes", "y"}
|
||||||
|
return bool(value)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_gate_id(value: Any) -> str:
|
||||||
|
if not isinstance(value, str) or not value.strip():
|
||||||
|
raise ManifestError("Every gate must define a non-empty gate_id.")
|
||||||
|
gate_id = value.strip()
|
||||||
|
if any(separator in gate_id for separator in ("/", "\\", os.sep)):
|
||||||
|
raise ManifestError(f"gate_id must not contain path separators: {gate_id}")
|
||||||
|
return gate_id
|
||||||
|
|
||||||
|
|
||||||
|
def validate_command(value: Any, gate_id: str) -> str | list[str]:
|
||||||
|
if isinstance(value, str) and value.strip():
|
||||||
|
return value
|
||||||
|
if isinstance(value, list) and value and all(isinstance(item, str) and item for item in value):
|
||||||
|
return value
|
||||||
|
raise ManifestError(f"Gate {gate_id} must define command as a non-empty string or string list.")
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_under_root(base: pathlib.Path, raw_path: Any, default: str, label: str) -> pathlib.Path:
|
||||||
|
candidate_text = default if raw_path in {None, ""} else str(raw_path)
|
||||||
|
candidate = pathlib.Path(candidate_text)
|
||||||
|
if not candidate.is_absolute():
|
||||||
|
candidate = base / candidate
|
||||||
|
resolved = candidate.resolve()
|
||||||
|
try:
|
||||||
|
resolved.relative_to(base)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise ManifestError(f"{label} must resolve under project_root: {candidate_text}") from exc
|
||||||
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_log_file(output_dir: pathlib.Path, raw_path: Any, gate_id: str) -> pathlib.Path:
|
||||||
|
candidate_text = f"logs/{gate_id}.log" if raw_path in {None, ""} else str(raw_path)
|
||||||
|
candidate = pathlib.Path(candidate_text)
|
||||||
|
if candidate.is_absolute():
|
||||||
|
resolved = candidate.resolve()
|
||||||
|
else:
|
||||||
|
resolved = (output_dir / candidate).resolve()
|
||||||
|
try:
|
||||||
|
resolved.relative_to(output_dir)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise ManifestError(f"log_file must resolve under output_dir for gate {gate_id}: {candidate_text}") from exc
|
||||||
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
|
def parse_gates(
|
||||||
|
raw_manifest: Any,
|
||||||
|
project_root: pathlib.Path,
|
||||||
|
output_dir: pathlib.Path,
|
||||||
|
) -> tuple[list[Gate], list[str], list[str]]:
|
||||||
|
gates: list[Gate] = []
|
||||||
|
errors: list[str] = []
|
||||||
|
declared: list[str] = []
|
||||||
|
for index, raw_gate in enumerate(manifest_gates(raw_manifest)):
|
||||||
|
try:
|
||||||
|
gate_id = validate_gate_id(raw_gate.get("gate_id"))
|
||||||
|
declared.append(gate_id)
|
||||||
|
command = validate_command(raw_gate.get("command"), gate_id)
|
||||||
|
working_directory = resolve_under_root(
|
||||||
|
project_root,
|
||||||
|
raw_gate.get("working_directory"),
|
||||||
|
".",
|
||||||
|
f"working_directory for gate {gate_id}",
|
||||||
|
)
|
||||||
|
expected_exit_code = int(raw_gate.get("expected_exit_code", 0))
|
||||||
|
timeout_seconds = float(raw_gate.get("timeout_seconds", 300))
|
||||||
|
if timeout_seconds <= 0:
|
||||||
|
raise ManifestError(f"timeout_seconds must be positive for gate {gate_id}.")
|
||||||
|
log_file = resolve_log_file(output_dir, raw_gate.get("log_file"), gate_id)
|
||||||
|
required = bool_value(raw_gate.get("required_before_review"), True)
|
||||||
|
gates.append(
|
||||||
|
Gate(
|
||||||
|
gate_id=gate_id,
|
||||||
|
command=command,
|
||||||
|
working_directory=working_directory,
|
||||||
|
expected_exit_code=expected_exit_code,
|
||||||
|
log_file=log_file,
|
||||||
|
timeout_seconds=timeout_seconds,
|
||||||
|
required_before_review=required,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except ManifestError as exc:
|
||||||
|
errors.append(f"gate[{index}]: {exc}")
|
||||||
|
return gates, errors, declared
|
||||||
|
|
||||||
|
|
||||||
|
def empty_report(project_root: pathlib.Path, manifest_path: pathlib.Path, mode: str) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"project_root": str(project_root),
|
||||||
|
"gate_manifest_path": str(manifest_path),
|
||||||
|
"mode": mode,
|
||||||
|
"commands_declared": [],
|
||||||
|
"commands_run": [],
|
||||||
|
"commands_skipped": [],
|
||||||
|
"exit_codes": {},
|
||||||
|
"pass_fail_status": "PENDING",
|
||||||
|
"duration": 0.0,
|
||||||
|
"log_paths": {},
|
||||||
|
"environment_notes": [
|
||||||
|
"Commands are declared by the manifest.",
|
||||||
|
"Passing gates are engineering evidence only; they do not imply product acceptance or lifecycle approval.",
|
||||||
|
],
|
||||||
|
"gate_results": [],
|
||||||
|
"manifest_errors": [],
|
||||||
|
"changed_files": [],
|
||||||
|
"machine_readable_summary": {},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def should_exclude(path: pathlib.Path, output_dir: pathlib.Path, project_root: pathlib.Path) -> bool:
|
||||||
|
try:
|
||||||
|
path.relative_to(output_dir)
|
||||||
|
return True
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
path.relative_to(project_root / ".git")
|
||||||
|
return True
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def snapshot_files(project_root: pathlib.Path, output_dir: pathlib.Path) -> dict[str, tuple[int, int]]:
|
||||||
|
snapshot: dict[str, tuple[int, int]] = {}
|
||||||
|
for path in project_root.rglob("*"):
|
||||||
|
if not path.is_file() or should_exclude(path, output_dir, project_root):
|
||||||
|
continue
|
||||||
|
stat = path.stat()
|
||||||
|
snapshot[path.relative_to(project_root).as_posix()] = (stat.st_size, stat.st_mtime_ns)
|
||||||
|
return snapshot
|
||||||
|
|
||||||
|
|
||||||
|
def diff_snapshots(before: dict[str, tuple[int, int]], after: dict[str, tuple[int, int]]) -> list[str]:
|
||||||
|
changed = {
|
||||||
|
path
|
||||||
|
for path in set(before) | set(after)
|
||||||
|
if before.get(path) != after.get(path)
|
||||||
|
}
|
||||||
|
return sorted(changed)
|
||||||
|
|
||||||
|
|
||||||
|
def command_display(command: str | list[str]) -> str:
|
||||||
|
if isinstance(command, list):
|
||||||
|
return " ".join(command)
|
||||||
|
return command
|
||||||
|
|
||||||
|
|
||||||
|
def command_for_subprocess(command: str | list[str]) -> tuple[str | list[str], bool]:
|
||||||
|
if isinstance(command, list):
|
||||||
|
return command, False
|
||||||
|
return command, True
|
||||||
|
|
||||||
|
|
||||||
|
def safe_text(value: Any) -> str:
|
||||||
|
if value is None:
|
||||||
|
return ""
|
||||||
|
if isinstance(value, bytes):
|
||||||
|
return value.decode("utf-8", errors="replace")
|
||||||
|
return str(value)
|
||||||
|
|
||||||
|
|
||||||
|
def run_gate(
|
||||||
|
gate: Gate,
|
||||||
|
project_root: pathlib.Path,
|
||||||
|
output_dir: pathlib.Path,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
start = time.monotonic()
|
||||||
|
before = snapshot_files(project_root, output_dir)
|
||||||
|
command, shell = command_for_subprocess(gate.command)
|
||||||
|
status = "PENDING"
|
||||||
|
exit_code: int | None = None
|
||||||
|
stdout = ""
|
||||||
|
stderr = ""
|
||||||
|
timeout = False
|
||||||
|
|
||||||
|
try:
|
||||||
|
completed = subprocess.run(
|
||||||
|
command,
|
||||||
|
cwd=str(gate.working_directory),
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
encoding="utf-8",
|
||||||
|
errors="replace",
|
||||||
|
timeout=gate.timeout_seconds,
|
||||||
|
shell=shell,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
exit_code = completed.returncode
|
||||||
|
stdout = completed.stdout
|
||||||
|
stderr = completed.stderr
|
||||||
|
status = "PASS" if exit_code == gate.expected_exit_code else "FAIL"
|
||||||
|
except subprocess.TimeoutExpired as exc:
|
||||||
|
timeout = True
|
||||||
|
stdout = safe_text(exc.stdout or exc.output)
|
||||||
|
stderr = safe_text(exc.stderr)
|
||||||
|
status = "TIMEOUT"
|
||||||
|
duration = time.monotonic() - start
|
||||||
|
after = snapshot_files(project_root, output_dir)
|
||||||
|
changed_files = diff_snapshots(before, after)
|
||||||
|
|
||||||
|
gate.log_file.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
log_lines = [
|
||||||
|
f"gate_id: {gate.gate_id}",
|
||||||
|
f"working_directory: {gate.working_directory}",
|
||||||
|
f"command: {command_display(gate.command)}",
|
||||||
|
f"expected_exit_code: {gate.expected_exit_code}",
|
||||||
|
f"exit_code: {exit_code}",
|
||||||
|
f"status: {status}",
|
||||||
|
f"duration_seconds: {duration:.3f}",
|
||||||
|
"",
|
||||||
|
"## stdout",
|
||||||
|
stdout,
|
||||||
|
"",
|
||||||
|
"## stderr",
|
||||||
|
stderr,
|
||||||
|
]
|
||||||
|
if timeout:
|
||||||
|
log_lines.extend(["", f"Command timed out after {gate.timeout_seconds} seconds."])
|
||||||
|
gate.log_file.write_text("\n".join(log_lines), encoding="utf-8")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"gate_id": gate.gate_id,
|
||||||
|
"command": command_display(gate.command),
|
||||||
|
"working_directory": str(gate.working_directory),
|
||||||
|
"expected_exit_code": gate.expected_exit_code,
|
||||||
|
"exit_code": exit_code,
|
||||||
|
"status": status,
|
||||||
|
"duration": duration,
|
||||||
|
"log_path": gate.log_file.relative_to(output_dir).as_posix(),
|
||||||
|
"required_before_review": gate.required_before_review,
|
||||||
|
"timeout": timeout,
|
||||||
|
"changed_files": changed_files,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def finalize_report(report: dict[str, Any]) -> None:
|
||||||
|
if report["manifest_errors"]:
|
||||||
|
status = "ERROR"
|
||||||
|
elif report["mode"] == "dry_run":
|
||||||
|
status = "DRY_RUN"
|
||||||
|
elif any(result["status"] in {"FAIL", "TIMEOUT"} for result in report["gate_results"]):
|
||||||
|
status = "FAIL"
|
||||||
|
else:
|
||||||
|
status = "PASS"
|
||||||
|
report["pass_fail_status"] = status
|
||||||
|
report["duration"] = round(float(report["duration"]), 3)
|
||||||
|
report["changed_files"] = sorted(
|
||||||
|
{
|
||||||
|
changed
|
||||||
|
for result in report["gate_results"]
|
||||||
|
for changed in result.get("changed_files", [])
|
||||||
|
}
|
||||||
|
)
|
||||||
|
report["machine_readable_summary"] = {
|
||||||
|
"status": status,
|
||||||
|
"commands_declared_count": len(report["commands_declared"]),
|
||||||
|
"commands_run_count": len(report["commands_run"]),
|
||||||
|
"commands_skipped_count": len(report["commands_skipped"]),
|
||||||
|
"manifest_error_count": len(report["manifest_errors"]),
|
||||||
|
"failed_count": sum(1 for result in report["gate_results"] if result["status"] == "FAIL"),
|
||||||
|
"timeout_count": sum(1 for result in report["gate_results"] if result["status"] == "TIMEOUT"),
|
||||||
|
"changed_file_count": len(report["changed_files"]),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def write_json_report(report: dict[str, Any], output_dir: pathlib.Path) -> None:
|
||||||
|
(output_dir / REPORT_JSON).write_text(
|
||||||
|
json.dumps(report, ensure_ascii=False, indent=2),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def write_markdown_report(report: dict[str, Any], output_dir: pathlib.Path) -> None:
|
||||||
|
summary = report["machine_readable_summary"]
|
||||||
|
lines = [
|
||||||
|
"# Gate Run Report",
|
||||||
|
"",
|
||||||
|
"## Summary",
|
||||||
|
"",
|
||||||
|
f"- Status: `{summary['status']}`",
|
||||||
|
f"- Commands declared: {summary['commands_declared_count']}",
|
||||||
|
f"- Commands run: {summary['commands_run_count']}",
|
||||||
|
f"- Commands skipped: {summary['commands_skipped_count']}",
|
||||||
|
f"- Manifest errors: {summary['manifest_error_count']}",
|
||||||
|
f"- Changed files: {summary['changed_file_count']}",
|
||||||
|
"",
|
||||||
|
"## Gate Results",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
if report["gate_results"]:
|
||||||
|
for result in report["gate_results"]:
|
||||||
|
lines.append(
|
||||||
|
f"- `{result['gate_id']}`: `{result['status']}` exit `{result['exit_code']}` log `{result['log_path']}`"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
lines.append("- None")
|
||||||
|
lines.extend(["", "## Skipped Gates", ""])
|
||||||
|
if report["commands_skipped"]:
|
||||||
|
for skipped in report["commands_skipped"]:
|
||||||
|
lines.append(f"- `{skipped['gate_id']}`: {skipped['reason']}")
|
||||||
|
else:
|
||||||
|
lines.append("- None")
|
||||||
|
lines.extend(["", "## Manifest Errors", ""])
|
||||||
|
if report["manifest_errors"]:
|
||||||
|
for error in report["manifest_errors"]:
|
||||||
|
lines.append(f"- {error}")
|
||||||
|
else:
|
||||||
|
lines.append("- None")
|
||||||
|
lines.extend(["", "## Changed Files", ""])
|
||||||
|
if report["changed_files"]:
|
||||||
|
for path in report["changed_files"]:
|
||||||
|
lines.append(f"- `{path}`")
|
||||||
|
else:
|
||||||
|
lines.append("- None")
|
||||||
|
(output_dir / REPORT_MD).write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def write_reports(report: dict[str, Any], output_dir: pathlib.Path) -> None:
|
||||||
|
output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
write_json_report(report, output_dir)
|
||||||
|
write_markdown_report(report, output_dir)
|
||||||
|
|
||||||
|
|
||||||
|
def run(
|
||||||
|
project_root: pathlib.Path,
|
||||||
|
manifest_path: pathlib.Path,
|
||||||
|
output_dir: pathlib.Path,
|
||||||
|
mode: str,
|
||||||
|
include_optional: bool = False,
|
||||||
|
) -> int:
|
||||||
|
project_root = project_root.resolve()
|
||||||
|
manifest_path = manifest_path.resolve()
|
||||||
|
output_dir = output_dir.resolve()
|
||||||
|
output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
report = empty_report(project_root, manifest_path, mode)
|
||||||
|
report["environment_notes"].extend(
|
||||||
|
[
|
||||||
|
f"python: {sys.version.split()[0]}",
|
||||||
|
f"platform: {platform.platform()}",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
if not project_root.is_dir():
|
||||||
|
report["manifest_errors"].append(f"project_root is not a directory: {project_root}")
|
||||||
|
finalize_report(report)
|
||||||
|
write_reports(report, output_dir)
|
||||||
|
return 2
|
||||||
|
if not manifest_path.is_file():
|
||||||
|
report["manifest_errors"].append(f"gate_manifest_path does not exist: {manifest_path}")
|
||||||
|
finalize_report(report)
|
||||||
|
write_reports(report, output_dir)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
try:
|
||||||
|
raw_manifest = load_raw_manifest(manifest_path)
|
||||||
|
gates, errors, declared = parse_gates(raw_manifest, project_root, output_dir)
|
||||||
|
report["commands_declared"] = declared
|
||||||
|
report["manifest_errors"].extend(errors)
|
||||||
|
except Exception as exc:
|
||||||
|
report["manifest_errors"].append(str(exc))
|
||||||
|
finalize_report(report)
|
||||||
|
write_reports(report, output_dir)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
if report["manifest_errors"]:
|
||||||
|
finalize_report(report)
|
||||||
|
write_reports(report, output_dir)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
run_start = time.monotonic()
|
||||||
|
for gate in gates:
|
||||||
|
if mode == "dry_run":
|
||||||
|
report["commands_skipped"].append({"gate_id": gate.gate_id, "reason": "dry_run"})
|
||||||
|
continue
|
||||||
|
if not gate.required_before_review and not include_optional:
|
||||||
|
report["commands_skipped"].append({"gate_id": gate.gate_id, "reason": "optional"})
|
||||||
|
continue
|
||||||
|
result = run_gate(gate, project_root, output_dir)
|
||||||
|
report["gate_results"].append(result)
|
||||||
|
report["commands_run"].append(gate.gate_id)
|
||||||
|
report["exit_codes"][gate.gate_id] = result["exit_code"]
|
||||||
|
report["log_paths"][gate.gate_id] = result["log_path"]
|
||||||
|
|
||||||
|
report["duration"] = time.monotonic() - run_start
|
||||||
|
finalize_report(report)
|
||||||
|
write_reports(report, output_dir)
|
||||||
|
return 0 if report["pass_fail_status"] in {"PASS", "DRY_RUN"} else 1
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
parser = argparse.ArgumentParser(description="Run or dry-run declared regression validation gates.")
|
||||||
|
parser.add_argument("--project-root", required=True, type=pathlib.Path)
|
||||||
|
parser.add_argument("--gate-manifest", required=True, type=pathlib.Path)
|
||||||
|
parser.add_argument("--output-dir", required=True, type=pathlib.Path)
|
||||||
|
parser.add_argument("--mode", choices=["dry_run", "run"], required=True)
|
||||||
|
parser.add_argument("--include-optional", action="store_true")
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
return run(
|
||||||
|
project_root=args.project_root,
|
||||||
|
manifest_path=args.gate_manifest,
|
||||||
|
output_dir=args.output_dir,
|
||||||
|
mode=args.mode,
|
||||||
|
include_optional=args.include_optional,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
|
|
@ -0,0 +1,306 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import pathlib
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import textwrap
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
|
||||||
|
SKILL_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||||
|
REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]
|
||||||
|
SCRIPT = SKILL_ROOT / "scripts" / "regression_validation_gate_runner.py"
|
||||||
|
TMP_ROOT = REPO_ROOT / "tmp" / "regression-validation-gate-runner-tests"
|
||||||
|
|
||||||
|
|
||||||
|
def quote(value: str) -> str:
|
||||||
|
return json.dumps(value)
|
||||||
|
|
||||||
|
|
||||||
|
def py_command(source: str) -> list[str]:
|
||||||
|
return [sys.executable, "-c", source]
|
||||||
|
|
||||||
|
|
||||||
|
def run_cli(*args: str) -> subprocess.CompletedProcess[str]:
|
||||||
|
return subprocess.run(
|
||||||
|
[sys.executable, str(SCRIPT), *args],
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def write_text(path: pathlib.Path, text: str) -> pathlib.Path:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text(text, encoding="utf-8")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def write_json(path: pathlib.Path, data: object) -> pathlib.Path:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def load_report(output_dir: pathlib.Path) -> dict[str, object]:
|
||||||
|
return json.loads((output_dir / "gate-run-report.json").read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
class RegressionValidationGateRunnerTest(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
TMP_ROOT.mkdir(parents=True, exist_ok=True)
|
||||||
|
self.tmp_dir = tempfile.TemporaryDirectory(dir=TMP_ROOT)
|
||||||
|
self.root = pathlib.Path(self.tmp_dir.name) / "project"
|
||||||
|
self.output_dir = pathlib.Path(self.tmp_dir.name) / "outputs"
|
||||||
|
self.manifest = pathlib.Path(self.tmp_dir.name) / "gates.json"
|
||||||
|
self.root.mkdir()
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.tmp_dir.cleanup()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls) -> None:
|
||||||
|
if TMP_ROOT.exists():
|
||||||
|
shutil.rmtree(TMP_ROOT)
|
||||||
|
|
||||||
|
def run_gate_runner(self, mode: str = "run") -> tuple[subprocess.CompletedProcess[str], dict[str, object]]:
|
||||||
|
result = run_cli(
|
||||||
|
"--project-root",
|
||||||
|
str(self.root),
|
||||||
|
"--gate-manifest",
|
||||||
|
str(self.manifest),
|
||||||
|
"--output-dir",
|
||||||
|
str(self.output_dir),
|
||||||
|
"--mode",
|
||||||
|
mode,
|
||||||
|
)
|
||||||
|
return result, load_report(self.output_dir)
|
||||||
|
|
||||||
|
def test_dry_run_parses_json_manifest_without_running_commands(self) -> None:
|
||||||
|
marker = self.root / "should-not-exist.txt"
|
||||||
|
write_json(
|
||||||
|
self.manifest,
|
||||||
|
{
|
||||||
|
"gates": [
|
||||||
|
{
|
||||||
|
"gate_id": "unit",
|
||||||
|
"command": py_command(f"open({quote(str(marker))}, 'w').write('ran')"),
|
||||||
|
"working_directory": ".",
|
||||||
|
"expected_exit_code": 0,
|
||||||
|
"timeout_seconds": 5,
|
||||||
|
"required_before_review": True,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
result, report = self.run_gate_runner(mode="dry_run")
|
||||||
|
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
self.assertFalse(marker.exists())
|
||||||
|
self.assertEqual(report["commands_declared"], ["unit"])
|
||||||
|
self.assertEqual(report["commands_run"], [])
|
||||||
|
self.assertEqual(report["commands_skipped"][0]["reason"], "dry_run")
|
||||||
|
self.assertEqual(report["machine_readable_summary"]["status"], "DRY_RUN")
|
||||||
|
|
||||||
|
def test_successful_command_capture_writes_log_and_passes(self) -> None:
|
||||||
|
write_json(
|
||||||
|
self.manifest,
|
||||||
|
{
|
||||||
|
"gates": [
|
||||||
|
{
|
||||||
|
"gate_id": "unit",
|
||||||
|
"command": py_command("print('hello gate')"),
|
||||||
|
"working_directory": ".",
|
||||||
|
"expected_exit_code": 0,
|
||||||
|
"log_file": "logs/unit.log",
|
||||||
|
"timeout_seconds": 5,
|
||||||
|
"required_before_review": True,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
result, report = self.run_gate_runner()
|
||||||
|
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
self.assertEqual(report["pass_fail_status"], "PASS")
|
||||||
|
self.assertEqual(report["exit_codes"]["unit"], 0)
|
||||||
|
log_path = self.output_dir / "logs" / "unit.log"
|
||||||
|
self.assertTrue(log_path.exists())
|
||||||
|
self.assertIn("hello gate", log_path.read_text(encoding="utf-8"))
|
||||||
|
self.assertEqual(report["commands_run"], ["unit"])
|
||||||
|
|
||||||
|
def test_failing_command_capture_fails_required_gate(self) -> None:
|
||||||
|
write_json(
|
||||||
|
self.manifest,
|
||||||
|
{
|
||||||
|
"gates": [
|
||||||
|
{
|
||||||
|
"gate_id": "regression",
|
||||||
|
"command": py_command("import sys; print('bad'); sys.exit(3)"),
|
||||||
|
"expected_exit_code": 0,
|
||||||
|
"timeout_seconds": 5,
|
||||||
|
"required_before_review": True,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
result, report = self.run_gate_runner()
|
||||||
|
|
||||||
|
self.assertEqual(result.returncode, 1, result.stderr)
|
||||||
|
self.assertEqual(report["pass_fail_status"], "FAIL")
|
||||||
|
self.assertEqual(report["exit_codes"]["regression"], 3)
|
||||||
|
gate = report["gate_results"][0]
|
||||||
|
self.assertEqual(gate["status"], "FAIL")
|
||||||
|
self.assertIn("bad", (self.output_dir / "logs" / "regression.log").read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
def test_optional_gate_is_skipped_by_default(self) -> None:
|
||||||
|
marker = self.root / "optional-ran.txt"
|
||||||
|
write_json(
|
||||||
|
self.manifest,
|
||||||
|
{
|
||||||
|
"gates": [
|
||||||
|
{
|
||||||
|
"gate_id": "optional-packaging",
|
||||||
|
"command": py_command(f"open({quote(str(marker))}, 'w').write('ran')"),
|
||||||
|
"expected_exit_code": 0,
|
||||||
|
"timeout_seconds": 5,
|
||||||
|
"required_before_review": False,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
result, report = self.run_gate_runner()
|
||||||
|
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
self.assertFalse(marker.exists())
|
||||||
|
self.assertEqual(report["commands_run"], [])
|
||||||
|
self.assertEqual(report["commands_skipped"][0]["gate_id"], "optional-packaging")
|
||||||
|
self.assertEqual(report["commands_skipped"][0]["reason"], "optional")
|
||||||
|
|
||||||
|
def test_missing_command_field_is_manifest_error(self) -> None:
|
||||||
|
write_json(
|
||||||
|
self.manifest,
|
||||||
|
{"gates": [{"gate_id": "broken", "expected_exit_code": 0, "required_before_review": True}]},
|
||||||
|
)
|
||||||
|
|
||||||
|
result, report = self.run_gate_runner()
|
||||||
|
|
||||||
|
self.assertEqual(result.returncode, 2)
|
||||||
|
self.assertEqual(report["pass_fail_status"], "ERROR")
|
||||||
|
self.assertEqual(report["machine_readable_summary"]["manifest_error_count"], 1)
|
||||||
|
self.assertIn("command", report["manifest_errors"][0])
|
||||||
|
|
||||||
|
def test_timeout_handling_marks_required_gate_failed(self) -> None:
|
||||||
|
write_json(
|
||||||
|
self.manifest,
|
||||||
|
{
|
||||||
|
"gates": [
|
||||||
|
{
|
||||||
|
"gate_id": "slow",
|
||||||
|
"command": py_command("import time; time.sleep(3)"),
|
||||||
|
"expected_exit_code": 0,
|
||||||
|
"timeout_seconds": 1,
|
||||||
|
"required_before_review": True,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
result, report = self.run_gate_runner()
|
||||||
|
|
||||||
|
self.assertEqual(result.returncode, 1, result.stderr)
|
||||||
|
gate = report["gate_results"][0]
|
||||||
|
self.assertEqual(gate["status"], "TIMEOUT")
|
||||||
|
self.assertEqual(report["pass_fail_status"], "FAIL")
|
||||||
|
self.assertIn("timed out", (self.output_dir / "logs" / "slow.log").read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
def test_markdown_manifest_with_yaml_fence_is_supported(self) -> None:
|
||||||
|
self.manifest = pathlib.Path(self.tmp_dir.name) / "gates.md"
|
||||||
|
write_text(
|
||||||
|
self.manifest,
|
||||||
|
textwrap.dedent(
|
||||||
|
f"""\
|
||||||
|
# Gates
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
gates:
|
||||||
|
- gate_id: markdown-gate
|
||||||
|
command:
|
||||||
|
- {sys.executable}
|
||||||
|
- -c
|
||||||
|
- "print('from markdown')"
|
||||||
|
expected_exit_code: 0
|
||||||
|
timeout_seconds: 5
|
||||||
|
required_before_review: true
|
||||||
|
```
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
result, report = self.run_gate_runner()
|
||||||
|
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
self.assertEqual(report["commands_declared"], ["markdown-gate"])
|
||||||
|
self.assertEqual(report["commands_run"], ["markdown-gate"])
|
||||||
|
|
||||||
|
def test_utf8_paths_and_chinese_filenames_are_logged(self) -> None:
|
||||||
|
project = pathlib.Path(self.tmp_dir.name) / "项目"
|
||||||
|
project.mkdir()
|
||||||
|
self.root = project
|
||||||
|
write_json(
|
||||||
|
self.manifest,
|
||||||
|
{
|
||||||
|
"gates": [
|
||||||
|
{
|
||||||
|
"gate_id": "中文-gate",
|
||||||
|
"command": py_command("print('中文输出')"),
|
||||||
|
"working_directory": ".",
|
||||||
|
"expected_exit_code": 0,
|
||||||
|
"timeout_seconds": 5,
|
||||||
|
"required_before_review": True,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
result, report = self.run_gate_runner()
|
||||||
|
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
log_path = self.output_dir / "logs" / "中文-gate.log"
|
||||||
|
self.assertTrue(log_path.exists())
|
||||||
|
self.assertIn("中文输出", log_path.read_text(encoding="utf-8"))
|
||||||
|
self.assertIn("中文-gate", report["log_paths"])
|
||||||
|
|
||||||
|
def test_project_file_changes_are_reported(self) -> None:
|
||||||
|
changed = self.root / "generated.txt"
|
||||||
|
write_json(
|
||||||
|
self.manifest,
|
||||||
|
{
|
||||||
|
"gates": [
|
||||||
|
{
|
||||||
|
"gate_id": "side-effect",
|
||||||
|
"command": py_command(f"open({quote(str(changed))}, 'w', encoding='utf-8').write('new')"),
|
||||||
|
"expected_exit_code": 0,
|
||||||
|
"timeout_seconds": 5,
|
||||||
|
"required_before_review": True,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
result, report = self.run_gate_runner()
|
||||||
|
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
self.assertEqual(report["changed_files"], ["generated.txt"])
|
||||||
|
self.assertEqual(report["gate_results"][0]["changed_files"], ["generated.txt"])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Loading…
Reference in New Issue