diff --git a/docs/install-sync.md b/docs/install-sync.md index ee72cbb..fa3acd8 100644 --- a/docs/install-sync.md +++ b/docs/install-sync.md @@ -24,6 +24,7 @@ PowerShell: powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill bundle-zip powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill lifecycle-status-guard-scan powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill repair-markdown-citations +powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill regression-validation-gate-runner ``` Use `-Force` to back up and replace an existing installed copy: @@ -32,6 +33,7 @@ Use `-Force` to back up and replace an existing installed copy: powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill bundle-zip -Force powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill lifecycle-status-guard-scan -Force powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill repair-markdown-citations -Force +powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill regression-validation-gate-runner -Force ``` `-All` installs only registry rows with an installable status and `default` install policy. Optional Skills must be named explicitly. diff --git a/registry/skills-index.md b/registry/skills-index.md index 0a60a81..e06319f 100644 --- a/registry/skills-index.md +++ b/registry/skills-index.md @@ -17,6 +17,7 @@ Existing CCPE content is not migrated here. | `fix-title` | Shift Markdown ATX heading depth for pasted LLM replies before insertion under parent sections. | `C:\Users\wangq\Documents\Codex\knowledge-vault\skills\fix-title` | `installed` | `default` | `C:\Users\wangq\.agents\skills\fix-title` | none | | `lifecycle-status-guard-scan` | Scan configured Markdown, JSON, YAML, and text files for lifecycle/status overclaim candidates and missing local evidence markers. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\lifecycle-status-guard-scan` | `installed` | `default` | `C:\Users\wangq\.agents\skills\lifecycle-status-guard-scan` | none | | `repair-markdown-citations` | Repair ChatGPT/Deep Research Markdown citation tokens into standard footnotes and a deduplicated reference section using exact turn-ID metadata mapping. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\repair-markdown-citations` | `installed` | `default` | `C:\Users\wangq\.agents\skills\repair-markdown-citations` | none | +| `regression-validation-gate-runner` | Dry-run or execute project-declared regression and validation gates from a manifest while capturing logs, exit codes, durations, skipped gates, and changed-file notes. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\regression-validation-gate-runner` | `installed` | `default` | `C:\Users\wangq\.agents\skills\regression-validation-gate-runner` | none | | `voice-generation` | Generate spoken audio from Markdown scripts using the local `voice-gen` CLI and MiniMax `mmx`, including custom voice clone registration and voice registry management. | `D:\AI\ClaudeCode\voice-ge` | `installed` | `conditional` | `C:\Users\wangq\.agents\skills\voice-generation` | none | ## Status Values diff --git a/skills/regression-validation-gate-runner/README.md b/skills/regression-validation-gate-runner/README.md new file mode 100644 index 0000000..dbd653e --- /dev/null +++ b/skills/regression-validation-gate-runner/README.md @@ -0,0 +1,42 @@ +# Regression Validation Gate Runner + +Source Skill for running or dry-running declared regression and validation gates while capturing durable logs and machine-readable reports. + +The canonical entry point is `SKILL.md`; the deterministic runner is in `scripts/regression_validation_gate_runner.py`. + +## Original Source + +```text +Original source: C:\Users\wangq\Documents\Codex\ccpe-system\requirements\skills-vault\2026-06-19-regression-validation-gate-runner.md +Migration date: 2026-06-19 +Migration status: first public Skill implementation +Behavior preserved: manifest-declared command execution only; no review judgment or lifecycle approval +Known gaps: no project-specific gate presets beyond the example fixture +``` + +## Layout + +```text +SKILL.md +README.md +agents/openai.yaml +fixtures/gate-manifest.example.yaml +scripts/regression_validation_gate_runner.py +tests/test_regression_validation_gate_runner.py +``` + +## Usage + +```powershell +conda run -n skills-vault python .\scripts\regression_validation_gate_runner.py ` + --project-root C:\path\project ` + --gate-manifest C:\path\gate-manifest.yaml ` + --output-dir C:\path\helper-outputs ` + --mode dry_run +``` + +## Tests + +```powershell +conda run -n skills-vault python -B -m unittest discover -s skills/regression-validation-gate-runner/tests -v +``` diff --git a/skills/regression-validation-gate-runner/SKILL.md b/skills/regression-validation-gate-runner/SKILL.md new file mode 100644 index 0000000..05aee71 --- /dev/null +++ b/skills/regression-validation-gate-runner/SKILL.md @@ -0,0 +1,98 @@ +--- +name: regression-validation-gate-runner +description: Use when Codex needs to dry-run or execute project-declared regression, validation, test, lint, build, schema, export, or review gates from a JSON, YAML, or Markdown manifest and capture logs before review, release, handoff, CCRA/local review, or lifecycle-sensitive work. +--- + +# Regression Validation Gate Runner + +## Purpose + +Run or verify explicitly declared validation gates and write a durable evidence package. Use the bundled script instead of ad hoc terminal copying when a reviewer needs command declarations, exit codes, logs, durations, skipped gates, and changed-file notes. + +Passing gates are engineering evidence only. They do not imply product acceptance, lifecycle approval, Owner approval, or CCRA approval. + +## Command + +```powershell +conda run -n skills-vault python .\scripts\regression_validation_gate_runner.py ` + --project-root C:\path\project ` + --gate-manifest C:\path\gate-manifest.yaml ` + --output-dir C:\path\helper-outputs ` + --mode dry_run +``` + +Use `--mode run` only when the user has asked to execute the manifest commands. Add `--include-optional` only when optional gates should run. + +When installed under `C:\Users\wangq\.agents\skills\regression-validation-gate-runner`, run from that Skill directory or use the installed script path. + +## Manifest + +The manifest may be JSON, YAML, YML, or Markdown containing one fenced `yaml`, `yml`, or `json` block. + +```yaml +gates: + - gate_id: unit-tests + command: + - conda + - run + - -n + - skills-vault + - python + - -B + - -m + - unittest + - discover + - -s + - skills/example/tests + - -v + working_directory: . + expected_exit_code: 0 + log_file: logs/unit-tests.log + timeout_seconds: 120 + required_before_review: true +``` + +`command` may be a string or an argv list. Lists avoid shell quoting surprises on Windows. + +## Modes + +- `dry_run`: parse and validate the manifest, then report declared and skipped gates without executing commands. +- `run`: execute required gates. Optional gates are skipped unless `--include-optional` is passed. + +## Outputs + +The script writes under `output_dir`: + +```text +gate-run-report.md +gate-run-report.json +logs/.log +``` + +Reports include: + +- commands declared, run, and skipped +- exit codes and pass/fail/timeout status +- duration and log paths +- environment notes +- project files changed during each gate +- machine-readable summary + +Exit code is `0` for `PASS` or `DRY_RUN`, `1` for failed or timed-out required gates, and `2` for manifest/input errors. + +## Safety + +- Run only commands explicitly listed in the manifest. +- Keep `working_directory` under `project_root`. +- Write logs and reports only under `output_dir`. +- Do not install dependencies, invent commands, edit project files directly, or repair failures. +- Record changed project files if a command creates, modifies, or deletes files; do not hide side effects. + +## Validation + +After changing runner behavior, run: + +```powershell +conda run -n skills-vault python -B -m unittest discover -s skills/regression-validation-gate-runner/tests -v +conda run -n skills-vault python -B C:\Users\wangq\.codex\skills\.system\skill-creator\scripts\quick_validate.py skills/regression-validation-gate-runner +``` diff --git a/skills/regression-validation-gate-runner/agents/openai.yaml b/skills/regression-validation-gate-runner/agents/openai.yaml new file mode 100644 index 0000000..2071009 --- /dev/null +++ b/skills/regression-validation-gate-runner/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Regression Validation Gate Runner" + short_description: "Run declared validation gates with logs" + default_prompt: "Use $regression-validation-gate-runner to dry-run or execute declared project gates and capture review-ready logs." diff --git a/skills/regression-validation-gate-runner/fixtures/gate-manifest.example.yaml b/skills/regression-validation-gate-runner/fixtures/gate-manifest.example.yaml new file mode 100644 index 0000000..6140860 --- /dev/null +++ b/skills/regression-validation-gate-runner/fixtures/gate-manifest.example.yaml @@ -0,0 +1,27 @@ +gates: + - gate_id: unit-tests + command: + - conda + - run + - -n + - skills-vault + - python + - -B + - -m + - unittest + - discover + - -s + - skills/example/tests + - -v + working_directory: . + expected_exit_code: 0 + log_file: logs/unit-tests.log + timeout_seconds: 120 + required_before_review: true + - gate_id: packaging-check + command: "python -m build --sdist --wheel" + working_directory: . + expected_exit_code: 0 + log_file: logs/packaging-check.log + timeout_seconds: 300 + required_before_review: false diff --git a/skills/regression-validation-gate-runner/scripts/regression_validation_gate_runner.py b/skills/regression-validation-gate-runner/scripts/regression_validation_gate_runner.py new file mode 100644 index 0000000..b373358 --- /dev/null +++ b/skills/regression-validation-gate-runner/scripts/regression_validation_gate_runner.py @@ -0,0 +1,486 @@ +from __future__ import annotations + +import argparse +import json +import os +import pathlib +import platform +import re +import subprocess +import sys +import time +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from typing import Any + +import yaml + + +REPORT_JSON = "gate-run-report.json" +REPORT_MD = "gate-run-report.md" + + +class ManifestError(ValueError): + pass + + +@dataclass +class Gate: + gate_id: str + command: str | list[str] + working_directory: pathlib.Path + expected_exit_code: int + log_file: pathlib.Path + timeout_seconds: float + required_before_review: bool + + +def load_raw_manifest(path: pathlib.Path) -> Any: + text = path.read_text(encoding="utf-8") + suffix = path.suffix.lower() + if suffix == ".json": + return json.loads(text) + if suffix in {".yaml", ".yml"}: + return yaml.safe_load(text) + if suffix in {".md", ".markdown"}: + fenced = re.search(r"```(yaml|yml|json)\s*\n(.*?)\n```", text, flags=re.DOTALL | re.I) + if not fenced: + raise ManifestError("Markdown manifest must contain a fenced yaml, yml, or json block.") + language = fenced.group(1).casefold() + body = fenced.group(2) + return json.loads(body) if language == "json" else yaml.safe_load(body) + raise ManifestError("Gate manifest must be JSON, YAML, YML, or Markdown.") + + +def manifest_gates(raw: Any) -> list[Mapping[str, Any]]: + if isinstance(raw, Mapping): + gates = raw.get("gates") + else: + gates = raw + if not isinstance(gates, list): + raise ManifestError("Gate manifest must define a gates list.") + normalized: list[Mapping[str, Any]] = [] + for index, item in enumerate(gates): + if not isinstance(item, Mapping): + raise ManifestError(f"Gate at index {index} must be an object.") + normalized.append(item) + return normalized + + +def bool_value(value: Any, default: bool) -> bool: + if value is None: + return default + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().casefold() in {"1", "true", "yes", "y"} + return bool(value) + + +def validate_gate_id(value: Any) -> str: + if not isinstance(value, str) or not value.strip(): + raise ManifestError("Every gate must define a non-empty gate_id.") + gate_id = value.strip() + if any(separator in gate_id for separator in ("/", "\\", os.sep)): + raise ManifestError(f"gate_id must not contain path separators: {gate_id}") + return gate_id + + +def validate_command(value: Any, gate_id: str) -> str | list[str]: + if isinstance(value, str) and value.strip(): + return value + if isinstance(value, list) and value and all(isinstance(item, str) and item for item in value): + return value + raise ManifestError(f"Gate {gate_id} must define command as a non-empty string or string list.") + + +def resolve_under_root(base: pathlib.Path, raw_path: Any, default: str, label: str) -> pathlib.Path: + candidate_text = default if raw_path in {None, ""} else str(raw_path) + candidate = pathlib.Path(candidate_text) + if not candidate.is_absolute(): + candidate = base / candidate + resolved = candidate.resolve() + try: + resolved.relative_to(base) + except ValueError as exc: + raise ManifestError(f"{label} must resolve under project_root: {candidate_text}") from exc + return resolved + + +def resolve_log_file(output_dir: pathlib.Path, raw_path: Any, gate_id: str) -> pathlib.Path: + candidate_text = f"logs/{gate_id}.log" if raw_path in {None, ""} else str(raw_path) + candidate = pathlib.Path(candidate_text) + if candidate.is_absolute(): + resolved = candidate.resolve() + else: + resolved = (output_dir / candidate).resolve() + try: + resolved.relative_to(output_dir) + except ValueError as exc: + raise ManifestError(f"log_file must resolve under output_dir for gate {gate_id}: {candidate_text}") from exc + return resolved + + +def parse_gates( + raw_manifest: Any, + project_root: pathlib.Path, + output_dir: pathlib.Path, +) -> tuple[list[Gate], list[str], list[str]]: + gates: list[Gate] = [] + errors: list[str] = [] + declared: list[str] = [] + for index, raw_gate in enumerate(manifest_gates(raw_manifest)): + try: + gate_id = validate_gate_id(raw_gate.get("gate_id")) + declared.append(gate_id) + command = validate_command(raw_gate.get("command"), gate_id) + working_directory = resolve_under_root( + project_root, + raw_gate.get("working_directory"), + ".", + f"working_directory for gate {gate_id}", + ) + expected_exit_code = int(raw_gate.get("expected_exit_code", 0)) + timeout_seconds = float(raw_gate.get("timeout_seconds", 300)) + if timeout_seconds <= 0: + raise ManifestError(f"timeout_seconds must be positive for gate {gate_id}.") + log_file = resolve_log_file(output_dir, raw_gate.get("log_file"), gate_id) + required = bool_value(raw_gate.get("required_before_review"), True) + gates.append( + Gate( + gate_id=gate_id, + command=command, + working_directory=working_directory, + expected_exit_code=expected_exit_code, + log_file=log_file, + timeout_seconds=timeout_seconds, + required_before_review=required, + ) + ) + except ManifestError as exc: + errors.append(f"gate[{index}]: {exc}") + return gates, errors, declared + + +def empty_report(project_root: pathlib.Path, manifest_path: pathlib.Path, mode: str) -> dict[str, Any]: + return { + "project_root": str(project_root), + "gate_manifest_path": str(manifest_path), + "mode": mode, + "commands_declared": [], + "commands_run": [], + "commands_skipped": [], + "exit_codes": {}, + "pass_fail_status": "PENDING", + "duration": 0.0, + "log_paths": {}, + "environment_notes": [ + "Commands are declared by the manifest.", + "Passing gates are engineering evidence only; they do not imply product acceptance or lifecycle approval.", + ], + "gate_results": [], + "manifest_errors": [], + "changed_files": [], + "machine_readable_summary": {}, + } + + +def should_exclude(path: pathlib.Path, output_dir: pathlib.Path, project_root: pathlib.Path) -> bool: + try: + path.relative_to(output_dir) + return True + except ValueError: + pass + try: + path.relative_to(project_root / ".git") + return True + except ValueError: + return False + + +def snapshot_files(project_root: pathlib.Path, output_dir: pathlib.Path) -> dict[str, tuple[int, int]]: + snapshot: dict[str, tuple[int, int]] = {} + for path in project_root.rglob("*"): + if not path.is_file() or should_exclude(path, output_dir, project_root): + continue + stat = path.stat() + snapshot[path.relative_to(project_root).as_posix()] = (stat.st_size, stat.st_mtime_ns) + return snapshot + + +def diff_snapshots(before: dict[str, tuple[int, int]], after: dict[str, tuple[int, int]]) -> list[str]: + changed = { + path + for path in set(before) | set(after) + if before.get(path) != after.get(path) + } + return sorted(changed) + + +def command_display(command: str | list[str]) -> str: + if isinstance(command, list): + return " ".join(command) + return command + + +def command_for_subprocess(command: str | list[str]) -> tuple[str | list[str], bool]: + if isinstance(command, list): + return command, False + return command, True + + +def safe_text(value: Any) -> str: + if value is None: + return "" + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + return str(value) + + +def run_gate( + gate: Gate, + project_root: pathlib.Path, + output_dir: pathlib.Path, +) -> dict[str, Any]: + start = time.monotonic() + before = snapshot_files(project_root, output_dir) + command, shell = command_for_subprocess(gate.command) + status = "PENDING" + exit_code: int | None = None + stdout = "" + stderr = "" + timeout = False + + try: + completed = subprocess.run( + command, + cwd=str(gate.working_directory), + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=gate.timeout_seconds, + shell=shell, + check=False, + ) + exit_code = completed.returncode + stdout = completed.stdout + stderr = completed.stderr + status = "PASS" if exit_code == gate.expected_exit_code else "FAIL" + except subprocess.TimeoutExpired as exc: + timeout = True + stdout = safe_text(exc.stdout or exc.output) + stderr = safe_text(exc.stderr) + status = "TIMEOUT" + duration = time.monotonic() - start + after = snapshot_files(project_root, output_dir) + changed_files = diff_snapshots(before, after) + + gate.log_file.parent.mkdir(parents=True, exist_ok=True) + log_lines = [ + f"gate_id: {gate.gate_id}", + f"working_directory: {gate.working_directory}", + f"command: {command_display(gate.command)}", + f"expected_exit_code: {gate.expected_exit_code}", + f"exit_code: {exit_code}", + f"status: {status}", + f"duration_seconds: {duration:.3f}", + "", + "## stdout", + stdout, + "", + "## stderr", + stderr, + ] + if timeout: + log_lines.extend(["", f"Command timed out after {gate.timeout_seconds} seconds."]) + gate.log_file.write_text("\n".join(log_lines), encoding="utf-8") + + return { + "gate_id": gate.gate_id, + "command": command_display(gate.command), + "working_directory": str(gate.working_directory), + "expected_exit_code": gate.expected_exit_code, + "exit_code": exit_code, + "status": status, + "duration": duration, + "log_path": gate.log_file.relative_to(output_dir).as_posix(), + "required_before_review": gate.required_before_review, + "timeout": timeout, + "changed_files": changed_files, + } + + +def finalize_report(report: dict[str, Any]) -> None: + if report["manifest_errors"]: + status = "ERROR" + elif report["mode"] == "dry_run": + status = "DRY_RUN" + elif any(result["status"] in {"FAIL", "TIMEOUT"} for result in report["gate_results"]): + status = "FAIL" + else: + status = "PASS" + report["pass_fail_status"] = status + report["duration"] = round(float(report["duration"]), 3) + report["changed_files"] = sorted( + { + changed + for result in report["gate_results"] + for changed in result.get("changed_files", []) + } + ) + report["machine_readable_summary"] = { + "status": status, + "commands_declared_count": len(report["commands_declared"]), + "commands_run_count": len(report["commands_run"]), + "commands_skipped_count": len(report["commands_skipped"]), + "manifest_error_count": len(report["manifest_errors"]), + "failed_count": sum(1 for result in report["gate_results"] if result["status"] == "FAIL"), + "timeout_count": sum(1 for result in report["gate_results"] if result["status"] == "TIMEOUT"), + "changed_file_count": len(report["changed_files"]), + } + + +def write_json_report(report: dict[str, Any], output_dir: pathlib.Path) -> None: + (output_dir / REPORT_JSON).write_text( + json.dumps(report, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + +def write_markdown_report(report: dict[str, Any], output_dir: pathlib.Path) -> None: + summary = report["machine_readable_summary"] + lines = [ + "# Gate Run Report", + "", + "## Summary", + "", + f"- Status: `{summary['status']}`", + f"- Commands declared: {summary['commands_declared_count']}", + f"- Commands run: {summary['commands_run_count']}", + f"- Commands skipped: {summary['commands_skipped_count']}", + f"- Manifest errors: {summary['manifest_error_count']}", + f"- Changed files: {summary['changed_file_count']}", + "", + "## Gate Results", + "", + ] + if report["gate_results"]: + for result in report["gate_results"]: + lines.append( + f"- `{result['gate_id']}`: `{result['status']}` exit `{result['exit_code']}` log `{result['log_path']}`" + ) + else: + lines.append("- None") + lines.extend(["", "## Skipped Gates", ""]) + if report["commands_skipped"]: + for skipped in report["commands_skipped"]: + lines.append(f"- `{skipped['gate_id']}`: {skipped['reason']}") + else: + lines.append("- None") + lines.extend(["", "## Manifest Errors", ""]) + if report["manifest_errors"]: + for error in report["manifest_errors"]: + lines.append(f"- {error}") + else: + lines.append("- None") + lines.extend(["", "## Changed Files", ""]) + if report["changed_files"]: + for path in report["changed_files"]: + lines.append(f"- `{path}`") + else: + lines.append("- None") + (output_dir / REPORT_MD).write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def write_reports(report: dict[str, Any], output_dir: pathlib.Path) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + write_json_report(report, output_dir) + write_markdown_report(report, output_dir) + + +def run( + project_root: pathlib.Path, + manifest_path: pathlib.Path, + output_dir: pathlib.Path, + mode: str, + include_optional: bool = False, +) -> int: + project_root = project_root.resolve() + manifest_path = manifest_path.resolve() + output_dir = output_dir.resolve() + output_dir.mkdir(parents=True, exist_ok=True) + report = empty_report(project_root, manifest_path, mode) + report["environment_notes"].extend( + [ + f"python: {sys.version.split()[0]}", + f"platform: {platform.platform()}", + ] + ) + + if not project_root.is_dir(): + report["manifest_errors"].append(f"project_root is not a directory: {project_root}") + finalize_report(report) + write_reports(report, output_dir) + return 2 + if not manifest_path.is_file(): + report["manifest_errors"].append(f"gate_manifest_path does not exist: {manifest_path}") + finalize_report(report) + write_reports(report, output_dir) + return 2 + + try: + raw_manifest = load_raw_manifest(manifest_path) + gates, errors, declared = parse_gates(raw_manifest, project_root, output_dir) + report["commands_declared"] = declared + report["manifest_errors"].extend(errors) + except Exception as exc: + report["manifest_errors"].append(str(exc)) + finalize_report(report) + write_reports(report, output_dir) + return 2 + + if report["manifest_errors"]: + finalize_report(report) + write_reports(report, output_dir) + return 2 + + run_start = time.monotonic() + for gate in gates: + if mode == "dry_run": + report["commands_skipped"].append({"gate_id": gate.gate_id, "reason": "dry_run"}) + continue + if not gate.required_before_review and not include_optional: + report["commands_skipped"].append({"gate_id": gate.gate_id, "reason": "optional"}) + continue + result = run_gate(gate, project_root, output_dir) + report["gate_results"].append(result) + report["commands_run"].append(gate.gate_id) + report["exit_codes"][gate.gate_id] = result["exit_code"] + report["log_paths"][gate.gate_id] = result["log_path"] + + report["duration"] = time.monotonic() - run_start + finalize_report(report) + write_reports(report, output_dir) + return 0 if report["pass_fail_status"] in {"PASS", "DRY_RUN"} else 1 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Run or dry-run declared regression validation gates.") + parser.add_argument("--project-root", required=True, type=pathlib.Path) + parser.add_argument("--gate-manifest", required=True, type=pathlib.Path) + parser.add_argument("--output-dir", required=True, type=pathlib.Path) + parser.add_argument("--mode", choices=["dry_run", "run"], required=True) + parser.add_argument("--include-optional", action="store_true") + args = parser.parse_args(argv) + return run( + project_root=args.project_root, + manifest_path=args.gate_manifest, + output_dir=args.output_dir, + mode=args.mode, + include_optional=args.include_optional, + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/regression-validation-gate-runner/tests/test_regression_validation_gate_runner.py b/skills/regression-validation-gate-runner/tests/test_regression_validation_gate_runner.py new file mode 100644 index 0000000..6aab71c --- /dev/null +++ b/skills/regression-validation-gate-runner/tests/test_regression_validation_gate_runner.py @@ -0,0 +1,306 @@ +from __future__ import annotations + +import json +import pathlib +import shutil +import subprocess +import sys +import tempfile +import textwrap +import unittest + + +SKILL_ROOT = pathlib.Path(__file__).resolve().parents[1] +REPO_ROOT = pathlib.Path(__file__).resolve().parents[3] +SCRIPT = SKILL_ROOT / "scripts" / "regression_validation_gate_runner.py" +TMP_ROOT = REPO_ROOT / "tmp" / "regression-validation-gate-runner-tests" + + +def quote(value: str) -> str: + return json.dumps(value) + + +def py_command(source: str) -> list[str]: + return [sys.executable, "-c", source] + + +def run_cli(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(SCRIPT), *args], + text=True, + capture_output=True, + check=False, + ) + + +def write_text(path: pathlib.Path, text: str) -> pathlib.Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def write_json(path: pathlib.Path, data: object) -> pathlib.Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + return path + + +def load_report(output_dir: pathlib.Path) -> dict[str, object]: + return json.loads((output_dir / "gate-run-report.json").read_text(encoding="utf-8")) + + +class RegressionValidationGateRunnerTest(unittest.TestCase): + def setUp(self) -> None: + TMP_ROOT.mkdir(parents=True, exist_ok=True) + self.tmp_dir = tempfile.TemporaryDirectory(dir=TMP_ROOT) + self.root = pathlib.Path(self.tmp_dir.name) / "project" + self.output_dir = pathlib.Path(self.tmp_dir.name) / "outputs" + self.manifest = pathlib.Path(self.tmp_dir.name) / "gates.json" + self.root.mkdir() + + def tearDown(self) -> None: + self.tmp_dir.cleanup() + + @classmethod + def tearDownClass(cls) -> None: + if TMP_ROOT.exists(): + shutil.rmtree(TMP_ROOT) + + def run_gate_runner(self, mode: str = "run") -> tuple[subprocess.CompletedProcess[str], dict[str, object]]: + result = run_cli( + "--project-root", + str(self.root), + "--gate-manifest", + str(self.manifest), + "--output-dir", + str(self.output_dir), + "--mode", + mode, + ) + return result, load_report(self.output_dir) + + def test_dry_run_parses_json_manifest_without_running_commands(self) -> None: + marker = self.root / "should-not-exist.txt" + write_json( + self.manifest, + { + "gates": [ + { + "gate_id": "unit", + "command": py_command(f"open({quote(str(marker))}, 'w').write('ran')"), + "working_directory": ".", + "expected_exit_code": 0, + "timeout_seconds": 5, + "required_before_review": True, + } + ] + }, + ) + + result, report = self.run_gate_runner(mode="dry_run") + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(marker.exists()) + self.assertEqual(report["commands_declared"], ["unit"]) + self.assertEqual(report["commands_run"], []) + self.assertEqual(report["commands_skipped"][0]["reason"], "dry_run") + self.assertEqual(report["machine_readable_summary"]["status"], "DRY_RUN") + + def test_successful_command_capture_writes_log_and_passes(self) -> None: + write_json( + self.manifest, + { + "gates": [ + { + "gate_id": "unit", + "command": py_command("print('hello gate')"), + "working_directory": ".", + "expected_exit_code": 0, + "log_file": "logs/unit.log", + "timeout_seconds": 5, + "required_before_review": True, + } + ] + }, + ) + + result, report = self.run_gate_runner() + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(report["pass_fail_status"], "PASS") + self.assertEqual(report["exit_codes"]["unit"], 0) + log_path = self.output_dir / "logs" / "unit.log" + self.assertTrue(log_path.exists()) + self.assertIn("hello gate", log_path.read_text(encoding="utf-8")) + self.assertEqual(report["commands_run"], ["unit"]) + + def test_failing_command_capture_fails_required_gate(self) -> None: + write_json( + self.manifest, + { + "gates": [ + { + "gate_id": "regression", + "command": py_command("import sys; print('bad'); sys.exit(3)"), + "expected_exit_code": 0, + "timeout_seconds": 5, + "required_before_review": True, + } + ] + }, + ) + + result, report = self.run_gate_runner() + + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(report["pass_fail_status"], "FAIL") + self.assertEqual(report["exit_codes"]["regression"], 3) + gate = report["gate_results"][0] + self.assertEqual(gate["status"], "FAIL") + self.assertIn("bad", (self.output_dir / "logs" / "regression.log").read_text(encoding="utf-8")) + + def test_optional_gate_is_skipped_by_default(self) -> None: + marker = self.root / "optional-ran.txt" + write_json( + self.manifest, + { + "gates": [ + { + "gate_id": "optional-packaging", + "command": py_command(f"open({quote(str(marker))}, 'w').write('ran')"), + "expected_exit_code": 0, + "timeout_seconds": 5, + "required_before_review": False, + } + ] + }, + ) + + result, report = self.run_gate_runner() + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(marker.exists()) + self.assertEqual(report["commands_run"], []) + self.assertEqual(report["commands_skipped"][0]["gate_id"], "optional-packaging") + self.assertEqual(report["commands_skipped"][0]["reason"], "optional") + + def test_missing_command_field_is_manifest_error(self) -> None: + write_json( + self.manifest, + {"gates": [{"gate_id": "broken", "expected_exit_code": 0, "required_before_review": True}]}, + ) + + result, report = self.run_gate_runner() + + self.assertEqual(result.returncode, 2) + self.assertEqual(report["pass_fail_status"], "ERROR") + self.assertEqual(report["machine_readable_summary"]["manifest_error_count"], 1) + self.assertIn("command", report["manifest_errors"][0]) + + def test_timeout_handling_marks_required_gate_failed(self) -> None: + write_json( + self.manifest, + { + "gates": [ + { + "gate_id": "slow", + "command": py_command("import time; time.sleep(3)"), + "expected_exit_code": 0, + "timeout_seconds": 1, + "required_before_review": True, + } + ] + }, + ) + + result, report = self.run_gate_runner() + + self.assertEqual(result.returncode, 1, result.stderr) + gate = report["gate_results"][0] + self.assertEqual(gate["status"], "TIMEOUT") + self.assertEqual(report["pass_fail_status"], "FAIL") + self.assertIn("timed out", (self.output_dir / "logs" / "slow.log").read_text(encoding="utf-8")) + + def test_markdown_manifest_with_yaml_fence_is_supported(self) -> None: + self.manifest = pathlib.Path(self.tmp_dir.name) / "gates.md" + write_text( + self.manifest, + textwrap.dedent( + f"""\ + # Gates + + ```yaml + gates: + - gate_id: markdown-gate + command: + - {sys.executable} + - -c + - "print('from markdown')" + expected_exit_code: 0 + timeout_seconds: 5 + required_before_review: true + ``` + """ + ), + ) + + result, report = self.run_gate_runner() + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(report["commands_declared"], ["markdown-gate"]) + self.assertEqual(report["commands_run"], ["markdown-gate"]) + + def test_utf8_paths_and_chinese_filenames_are_logged(self) -> None: + project = pathlib.Path(self.tmp_dir.name) / "项目" + project.mkdir() + self.root = project + write_json( + self.manifest, + { + "gates": [ + { + "gate_id": "中文-gate", + "command": py_command("print('中文输出')"), + "working_directory": ".", + "expected_exit_code": 0, + "timeout_seconds": 5, + "required_before_review": True, + } + ] + }, + ) + + result, report = self.run_gate_runner() + + self.assertEqual(result.returncode, 0, result.stderr) + log_path = self.output_dir / "logs" / "中文-gate.log" + self.assertTrue(log_path.exists()) + self.assertIn("中文输出", log_path.read_text(encoding="utf-8")) + self.assertIn("中文-gate", report["log_paths"]) + + def test_project_file_changes_are_reported(self) -> None: + changed = self.root / "generated.txt" + write_json( + self.manifest, + { + "gates": [ + { + "gate_id": "side-effect", + "command": py_command(f"open({quote(str(changed))}, 'w', encoding='utf-8').write('new')"), + "expected_exit_code": 0, + "timeout_seconds": 5, + "required_before_review": True, + } + ] + }, + ) + + result, report = self.run_gate_runner() + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(report["changed_files"], ["generated.txt"]) + self.assertEqual(report["gate_results"][0]["changed_files"], ["generated.txt"]) + + +if __name__ == "__main__": + unittest.main()