diff --git a/docs/install-sync.md b/docs/install-sync.md index d92dcb1..52a6df7 100644 --- a/docs/install-sync.md +++ b/docs/install-sync.md @@ -27,6 +27,7 @@ powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill repa powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill regression-validation-gate-runner powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill review-bundle-audit powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill review-context-builder +powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill routing-behavior-diff-audit ``` Use `-Force` to back up and replace an existing installed copy: @@ -38,6 +39,7 @@ powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill repa powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill regression-validation-gate-runner -Force powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill review-bundle-audit -Force powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill review-context-builder -Force +powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill routing-behavior-diff-audit -Force ``` `-All` installs only registry rows with an installable status and `default` install policy. Optional Skills must be named explicitly. diff --git a/registry/skills-index.md b/registry/skills-index.md index 379292a..9bb0479 100644 --- a/registry/skills-index.md +++ b/registry/skills-index.md @@ -20,6 +20,7 @@ Existing CCPE content is not migrated here. | `regression-validation-gate-runner` | Dry-run or execute project-declared regression and validation gates from a manifest while capturing logs, exit codes, durations, skipped gates, and changed-file notes. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\regression-validation-gate-runner` | `installed` | `default` | `C:\Users\wangq\.agents\skills\regression-validation-gate-runner` | none | | `review-bundle-audit` | Preflight review, handoff, release, CCRA, Web upload, or code-review bundle directories for required files, manifests, sidecars, reports, zip readability, warnings, and blockers. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\review-bundle-audit` | `installed` | `default` | `C:\Users\wangq\.agents\skills\review-bundle-audit` | none | | `review-context-builder` | Build file-first review, audit, planning, release, handoff, PR, or Agent invocation context indexes and manifests from configured local source roots, patterns, and metadata. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\review-context-builder` | `installed` | `default` | `C:\Users\wangq\.agents\skills\review-context-builder` | none | +| `routing-behavior-diff-audit` | Compare before/after selector, classifier, or routing outputs for targeted changes, collateral non-target changes, no-call changes, missing cases, and expected-route mismatch changes. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\routing-behavior-diff-audit` | `installed` | `default` | `C:\Users\wangq\.agents\skills\routing-behavior-diff-audit` | none | | `voice-generation` | Generate spoken audio from Markdown scripts using the local `voice-gen` CLI and MiniMax `mmx`, including custom voice clone registration and voice registry management. | `D:\AI\ClaudeCode\voice-ge` | `installed` | `conditional` | `C:\Users\wangq\.agents\skills\voice-generation` | none | ## Status Values diff --git a/skills/routing-behavior-diff-audit/README.md b/skills/routing-behavior-diff-audit/README.md new file mode 100644 index 0000000..fe75407 --- /dev/null +++ b/skills/routing-behavior-diff-audit/README.md @@ -0,0 +1,43 @@ +# Routing Behavior Diff Audit + +Source Skill for deterministic before/after routing result comparison. + +The canonical entry point is `SKILL.md`; the deterministic diff engine is in `scripts/routing_behavior_diff_audit.py`. + +## Original Source + +```text +Original source: C:\Users\wangq\Documents\Codex\ccpe-system\requirements\skills-vault\2026-06-19-routing-behavior-diff-audit.md +Migration date: 2026-06-19 +Migration status: first public Skill implementation +Behavior preserved: deterministic routing-result diff only; no selector edits, expected-label updates, or patch-closure judgment +Known gaps: Markdown support is limited to fenced JSON/YAML blocks; product-specific meaning remains outside this Skill +``` + +## Layout + +```text +SKILL.md +README.md +agents/openai.yaml +fixtures/before.example.json +fixtures/after.example.json +scripts/routing_behavior_diff_audit.py +tests/test_routing_behavior_diff_audit.py +``` + +## Usage + +```powershell +conda run -n skills-vault python .\scripts\routing_behavior_diff_audit.py ` + --before-results C:\path\before.json ` + --after-results C:\path\after.json ` + --output-dir C:\path\helper-outputs ` + --targeted-case CASE-001 +``` + +## Tests + +```powershell +conda run -n skills-vault python -B -m unittest discover -s skills/routing-behavior-diff-audit/tests -v +``` diff --git a/skills/routing-behavior-diff-audit/SKILL.md b/skills/routing-behavior-diff-audit/SKILL.md new file mode 100644 index 0000000..d0f5c16 --- /dev/null +++ b/skills/routing-behavior-diff-audit/SKILL.md @@ -0,0 +1,92 @@ +--- +name: routing-behavior-diff-audit +description: Use when Codex needs to compare before/after selector, routing, QPI, classifier, rules-engine, triage, assignment, or model-governance outputs to identify targeted route changes, collateral non-target changes, no-call changes, missing cases, and new or resolved expected-route mismatches before review, release, handoff, CCRA/local review, or regression closeout. +--- + +# Routing Behavior Diff Audit + +## Purpose + +Run a deterministic before/after behavior diff for rules-based routing outputs. Use the bundled script to compare case IDs, routes, optional expected routes, missing cases, no-call transitions, and targeted versus non-target changes. + +This Skill reports observed differences only. It must not edit routing rules, change regression cases, update expected labels, infer product meaning, or decide whether a patch should close. + +## Command + +```powershell +conda run -n skills-vault python .\scripts\routing_behavior_diff_audit.py ` + --before-results C:\path\before-results ` + --after-results C:\path\after-results ` + --output-dir C:\path\review-run ` + --case-id-field case_id ` + --route-field route ` + --expected-route-field expected_route ` + --targeted-case CASE-001 +``` + +When installed under `C:\Users\wangq\.agents\skills\routing-behavior-diff-audit`, run from that Skill directory or use the installed script path. + +## Inputs + +- `--before-results`: JSON, JSONL, or Markdown file, or a directory containing those files. +- `--after-results`: JSON, JSONL, or Markdown file, or a directory containing those files. +- `--output-dir`: directory where reports are written. +- `--case-id-field`: field containing the stable case ID. Default: `case_id`. +- `--route-field`: field containing the routing decision. Default: `route`. +- `--expected-route-field`: optional field containing the expected route for failure comparison. +- `--targeted-case`: repeatable case ID that the patch intentionally targets. +- `--targeted-cases-file`: optional JSON/YAML list, or object with `targeted_cases`, `case_ids`, or `cases`. + +JSON is the stable input target. Directory inputs are scanned recursively for `.json`, `.jsonl`, `.md`, and `.markdown`; Markdown support expects a fenced JSON or YAML block. + +## Outputs + +The script writes under `output_dir`: + +```text +routing-behavior-diff.md +routing-behavior-diff.json +``` + +The JSON report includes: + +- `total_cases_compared` +- `unchanged_cases` +- `changed_cases` +- `targeted_changes` +- `non_target_changes` +- `new_failures` +- `resolved_failures` +- `missing_before_cases` +- `missing_after_cases` +- `diff_table` +- `machine_readable_summary` + +`machine_readable_summary.status` is `PASS`, `ATTENTION`, or `FAIL` for automation routing only. It is not a patch approval, owner approval, CCRA decision, or release decision. + +## Comparison Rules + +- Compare only case IDs present in both before and after for `total_cases_compared`. +- Classify after-only IDs as `missing_before_cases`. +- Classify before-only IDs as `missing_after_cases`. +- Treat a common case as changed when the route, expected route, no-call status, or expected-route failure status changes. +- Classify changed cases listed by `--targeted-case` or `--targeted-cases-file` as `targeted_changes`. +- Classify all other changed cases as `non_target_changes`. +- Compute `new_failures` and `resolved_failures` only when `--expected-route-field` is supplied and present in the records. +- Treat empty, null-like, and common no-call spellings such as `no_call`, `no-call`, and `no call` as no-call values. + +## Safety + +- Read only configured before/after result paths and optional targeted-cases file. +- Write only `routing-behavior-diff.md` and `routing-behavior-diff.json` under `output_dir`. +- Do not edit selectors, routing rules, regression cases, expected labels, review bundles, model cards, or governance artifacts. +- Do not decide acceptance, lifecycle status, patch closure, release readiness, or owner approval. + +## Validation + +After changing diff behavior, run: + +```powershell +conda run -n skills-vault python -B -m unittest discover -s skills/routing-behavior-diff-audit/tests -v +conda run -n skills-vault python -B C:\Users\wangq\.codex\skills\.system\skill-creator\scripts\quick_validate.py skills/routing-behavior-diff-audit +``` diff --git a/skills/routing-behavior-diff-audit/agents/openai.yaml b/skills/routing-behavior-diff-audit/agents/openai.yaml new file mode 100644 index 0000000..6df2798 --- /dev/null +++ b/skills/routing-behavior-diff-audit/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Routing Behavior Diff Audit" + short_description: "Compare before/after routing outputs" + default_prompt: "Use $routing-behavior-diff-audit to compare before and after routing results and write routing-behavior-diff reports." diff --git a/skills/routing-behavior-diff-audit/fixtures/after.example.json b/skills/routing-behavior-diff-audit/fixtures/after.example.json new file mode 100644 index 0000000..d3f85cd --- /dev/null +++ b/skills/routing-behavior-diff-audit/fixtures/after.example.json @@ -0,0 +1,12 @@ +[ + { + "case_id": "CASE-001", + "route": "qpi-revised", + "expected_route": "qpi-revised" + }, + { + "case_id": "CASE-002", + "route": "review", + "expected_route": "review" + } +] diff --git a/skills/routing-behavior-diff-audit/fixtures/before.example.json b/skills/routing-behavior-diff-audit/fixtures/before.example.json new file mode 100644 index 0000000..0e704de --- /dev/null +++ b/skills/routing-behavior-diff-audit/fixtures/before.example.json @@ -0,0 +1,12 @@ +[ + { + "case_id": "CASE-001", + "route": "qpi", + "expected_route": "qpi" + }, + { + "case_id": "CASE-002", + "route": "no_call", + "expected_route": "review" + } +] diff --git a/skills/routing-behavior-diff-audit/scripts/routing_behavior_diff_audit.py b/skills/routing-behavior-diff-audit/scripts/routing_behavior_diff_audit.py new file mode 100644 index 0000000..dbf03ea --- /dev/null +++ b/skills/routing-behavior-diff-audit/scripts/routing_behavior_diff_audit.py @@ -0,0 +1,491 @@ +from __future__ import annotations + +import argparse +import json +import pathlib +import re +import sys +from typing import Any + + +RESULT_KEYS = ("results", "cases", "records", "items", "diff_table") +SUPPORTED_SUFFIXES = {".json", ".jsonl", ".md", ".markdown"} +NO_CALL_VALUES = {"", "none", "null", "no_call", "no-call", "no call", "nocall", "no decision"} + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + targeted_cases, targeted_errors = load_targeted_cases(args.targeted_case, args.targeted_cases_file) + before_cases, before_errors = load_case_map( + args.before_results, + args.case_id_field, + args.route_field, + args.expected_route_field, + "before", + ) + after_cases, after_errors = load_case_map( + args.after_results, + args.case_id_field, + args.route_field, + args.expected_route_field, + "after", + ) + blocking_errors = [*targeted_errors, *before_errors, *after_errors] + + if blocking_errors: + summary = empty_summary(blocking_errors) + write_reports(args.output_dir, summary) + print(f"routing-behavior-diff-audit: {len(blocking_errors)} blocking error(s)", file=sys.stderr) + return 1 + + summary = compare_cases(before_cases, after_cases, targeted_cases) + write_reports(args.output_dir, summary) + print(str(args.output_dir / "routing-behavior-diff.json")) + return 0 + + +def parse_args(argv: list[str] | None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Compare before/after rules-based routing outputs and write deterministic diff reports.", + ) + parser.add_argument("--before-results", required=True, type=pathlib.Path) + parser.add_argument("--after-results", required=True, type=pathlib.Path) + parser.add_argument("--output-dir", required=True, type=pathlib.Path) + parser.add_argument("--case-id-field", default="case_id") + parser.add_argument("--route-field", default="route") + parser.add_argument("--expected-route-field", default=None) + parser.add_argument("--targeted-case", action="append", default=[]) + parser.add_argument("--targeted-cases-file", type=pathlib.Path) + return parser.parse_args(argv) + + +def load_targeted_cases(raw_cases: list[str], cases_file: pathlib.Path | None) -> tuple[set[str], list[str]]: + targeted = {str(case_id) for case_id in raw_cases} + errors: list[str] = [] + if not cases_file: + return targeted, errors + if not cases_file.exists(): + return targeted, [f"Targeted cases file not found: {cases_file}"] + + try: + loaded = parse_file(cases_file) + except ValueError as exc: + return targeted, [f"Unable to parse targeted cases file {cases_file}: {exc}"] + + candidate: Any = loaded + if isinstance(candidate, dict): + for key in ("targeted_cases", "case_ids", "cases"): + if key in candidate: + candidate = candidate[key] + break + if not isinstance(candidate, list): + return targeted, [f"Targeted cases file must contain a list or object with targeted_cases/case_ids/cases: {cases_file}"] + + for item in candidate: + if isinstance(item, dict): + for key in ("case_id", "id"): + if key in item: + targeted.add(str(item[key])) + break + else: + targeted.add(str(item)) + return targeted, errors + + +def load_case_map( + root: pathlib.Path, + case_id_field: str, + route_field: str, + expected_route_field: str | None, + side: str, +) -> tuple[dict[str, dict[str, Any]], list[str]]: + errors: list[str] = [] + if not root.exists(): + return {}, [f"{side} results path not found: {root}"] + + files = collect_input_files(root) + if not files: + return {}, [f"{side} results path contains no supported files: {root}"] + + cases: dict[str, dict[str, Any]] = {} + for path in files: + try: + parsed = parse_file(path) + except ValueError as exc: + errors.append(f"Unable to parse {side} file {path}: {exc}") + continue + + records = extract_records(parsed) + if not records: + errors.append(f"No records found in {side} file: {path}") + continue + + for record in records: + if not isinstance(record, dict): + errors.append(f"Record in {side} file is not an object: {path}") + continue + if case_id_field not in record: + errors.append(f"Record missing case id field '{case_id_field}' in {side} file: {path}") + continue + case_id = stable_string(record[case_id_field]) + if case_id in cases: + errors.append(f"Duplicate case id '{case_id}' in {side} inputs") + continue + cases[case_id] = { + "case_id": case_id, + "route": normalize_value(record.get(route_field)), + "expected_route": normalize_value(record.get(expected_route_field)) if expected_route_field else None, + "source_file": str(path), + } + return cases, errors + + +def collect_input_files(root: pathlib.Path) -> list[pathlib.Path]: + if root.is_file(): + return [root] if root.suffix.lower() in SUPPORTED_SUFFIXES else [] + return sorted(path for path in root.rglob("*") if path.is_file() and path.suffix.lower() in SUPPORTED_SUFFIXES) + + +def parse_file(path: pathlib.Path) -> Any: + suffix = path.suffix.lower() + text = path.read_text(encoding="utf-8") + if suffix == ".json": + return json.loads(text) + if suffix == ".jsonl": + records = [] + for line_number, line in enumerate(text.splitlines(), start=1): + stripped = line.strip() + if not stripped: + continue + try: + records.append(json.loads(stripped)) + except json.JSONDecodeError as exc: + raise ValueError(f"invalid JSONL on line {line_number}: {exc}") from exc + return records + if suffix in {".md", ".markdown"}: + return parse_markdown_summary(text, path) + raise ValueError(f"unsupported suffix: {suffix}") + + +def parse_markdown_summary(text: str, path: pathlib.Path) -> Any: + fences = re.findall(r"```(?:json|yaml|yml)?\s*\n(.*?)\n```", text, flags=re.DOTALL | re.IGNORECASE) + errors: list[str] = [] + for fence in fences: + try: + return json.loads(fence) + except json.JSONDecodeError as exc: + errors.append(f"json: {exc}") + yaml_loaded = try_load_yaml(fence) + if yaml_loaded is not None: + return yaml_loaded + raise ValueError(f"no parseable JSON/YAML fenced block found in Markdown summary {path}; {'; '.join(errors)}") + + +def try_load_yaml(text: str) -> Any: + try: + import yaml # type: ignore[import-not-found] + except ImportError: + return None + try: + return yaml.safe_load(text) + except Exception: + return None + + +def extract_records(parsed: Any) -> list[Any]: + if isinstance(parsed, list): + return parsed + if isinstance(parsed, dict): + for key in RESULT_KEYS: + value = parsed.get(key) + if isinstance(value, list): + return value + if "case_id" in parsed or "id" in parsed: + return [parsed] + return [] + + +def compare_cases(before_cases: dict[str, dict[str, Any]], after_cases: dict[str, dict[str, Any]], targeted_cases: set[str]) -> dict[str, Any]: + before_ids = set(before_cases) + after_ids = set(after_cases) + common_ids = sorted(before_ids & after_ids) + missing_before = sorted(after_ids - before_ids) + missing_after = sorted(before_ids - after_ids) + + unchanged: list[str] = [] + changed: list[str] = [] + targeted_changes: list[str] = [] + non_target_changes: list[str] = [] + new_failures: list[str] = [] + resolved_failures: list[str] = [] + diff_table: list[dict[str, Any]] = [] + + for case_id in common_ids: + before = before_cases[case_id] + after = after_cases[case_id] + before_failure = is_failure(before) + after_failure = is_failure(after) + route_changed = before["route"] != after["route"] + expected_changed = before["expected_route"] != after["expected_route"] + failure_status_changed = before_failure != after_failure + no_call_changed = is_no_call(before["route"]) != is_no_call(after["route"]) + row_changed = route_changed or expected_changed or failure_status_changed or no_call_changed + change_type = build_change_type(route_changed, expected_changed, failure_status_changed, no_call_changed) + + if row_changed: + changed.append(case_id) + if case_id in targeted_cases: + targeted_changes.append(case_id) + else: + non_target_changes.append(case_id) + else: + unchanged.append(case_id) + + if not before_failure and after_failure: + new_failures.append(case_id) + if before_failure and not after_failure: + resolved_failures.append(case_id) + + diff_table.append( + { + "case_id": case_id, + "change_type": change_type, + "targeted": case_id in targeted_cases, + "before_route": before["route"], + "after_route": after["route"], + "before_expected_route": before["expected_route"], + "after_expected_route": after["expected_route"], + "route_changed": route_changed, + "expected_route_changed": expected_changed, + "before_failure": before_failure, + "after_failure": after_failure, + "failure_status_changed": failure_status_changed, + "no_call_changed": no_call_changed, + "before_source_file": before["source_file"], + "after_source_file": after["source_file"], + } + ) + + for case_id in missing_before: + after = after_cases[case_id] + diff_table.append( + { + "case_id": case_id, + "change_type": "missing_before", + "targeted": case_id in targeted_cases, + "before_route": None, + "after_route": after["route"], + "before_expected_route": None, + "after_expected_route": after["expected_route"], + "route_changed": True, + "expected_route_changed": after["expected_route"] is not None, + "before_failure": None, + "after_failure": is_failure(after), + "failure_status_changed": None, + "no_call_changed": None, + "before_source_file": None, + "after_source_file": after["source_file"], + } + ) + + for case_id in missing_after: + before = before_cases[case_id] + diff_table.append( + { + "case_id": case_id, + "change_type": "missing_after", + "targeted": case_id in targeted_cases, + "before_route": before["route"], + "after_route": None, + "before_expected_route": before["expected_route"], + "after_expected_route": None, + "route_changed": True, + "expected_route_changed": before["expected_route"] is not None, + "before_failure": is_failure(before), + "after_failure": None, + "failure_status_changed": None, + "no_call_changed": None, + "before_source_file": before["source_file"], + "after_source_file": None, + } + ) + + status = "ATTENTION" if non_target_changes or missing_before or missing_after or new_failures else "PASS" + return { + "total_cases_compared": len(common_ids), + "unchanged_cases": unchanged, + "changed_cases": changed, + "targeted_changes": targeted_changes, + "non_target_changes": non_target_changes, + "new_failures": new_failures, + "resolved_failures": resolved_failures, + "missing_before_cases": missing_before, + "missing_after_cases": missing_after, + "diff_table": sorted(diff_table, key=lambda row: row["case_id"]), + "blocking_errors": [], + "machine_readable_summary": { + "status": status, + "total_cases_compared": len(common_ids), + "changed_count": len(changed), + "targeted_change_count": len(targeted_changes), + "non_target_change_count": len(non_target_changes), + "new_failure_count": len(new_failures), + "resolved_failure_count": len(resolved_failures), + "missing_before_count": len(missing_before), + "missing_after_count": len(missing_after), + }, + } + + +def build_change_type(route_changed: bool, expected_changed: bool, failure_status_changed: bool, no_call_changed: bool) -> str: + labels: list[str] = [] + if route_changed: + labels.append("route_changed") + if expected_changed: + labels.append("expected_route_changed") + if failure_status_changed: + labels.append("failure_status_changed") + if no_call_changed: + labels.append("no_call_changed") + return "+".join(labels) if labels else "unchanged" + + +def is_failure(case: dict[str, Any]) -> bool: + expected = case.get("expected_route") + if expected is None: + return False + return case.get("route") != expected + + +def is_no_call(value: Any) -> bool: + if value is None: + return True + return str(value).strip().casefold() in NO_CALL_VALUES + + +def normalize_value(value: Any) -> str | None: + if value is None: + return None + return stable_string(value) + + +def stable_string(value: Any) -> str: + if isinstance(value, str): + return value + return json.dumps(value, ensure_ascii=False, sort_keys=True) + + +def empty_summary(blocking_errors: list[str]) -> dict[str, Any]: + return { + "total_cases_compared": 0, + "unchanged_cases": [], + "changed_cases": [], + "targeted_changes": [], + "non_target_changes": [], + "new_failures": [], + "resolved_failures": [], + "missing_before_cases": [], + "missing_after_cases": [], + "diff_table": [], + "blocking_errors": blocking_errors, + "machine_readable_summary": { + "status": "FAIL", + "blocking_error_count": len(blocking_errors), + "total_cases_compared": 0, + "changed_count": 0, + "targeted_change_count": 0, + "non_target_change_count": 0, + "new_failure_count": 0, + "resolved_failure_count": 0, + "missing_before_count": 0, + "missing_after_count": 0, + }, + } + + +def write_reports(output_dir: pathlib.Path, summary: dict[str, Any]) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "routing-behavior-diff.json").write_text( + json.dumps(summary, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + (output_dir / "routing-behavior-diff.md").write_text(render_markdown(summary), encoding="utf-8") + + +def render_markdown(summary: dict[str, Any]) -> str: + machine = summary["machine_readable_summary"] + lines = [ + "# Routing Behavior Diff", + "", + f"Status: `{machine['status']}`", + "", + "## Summary", + "", + f"- Total cases compared: {summary['total_cases_compared']}", + f"- Unchanged cases: {len(summary['unchanged_cases'])}", + f"- Changed cases: {len(summary['changed_cases'])}", + f"- Targeted changes: {len(summary['targeted_changes'])}", + f"- Non-target changes: {len(summary['non_target_changes'])}", + f"- New failures: {len(summary['new_failures'])}", + f"- Resolved failures: {len(summary['resolved_failures'])}", + f"- Missing before cases: {len(summary['missing_before_cases'])}", + f"- Missing after cases: {len(summary['missing_after_cases'])}", + "", + ] + if summary["blocking_errors"]: + lines.extend(["## Blocking Errors", ""]) + lines.extend(f"- {error}" for error in summary["blocking_errors"]) + lines.append("") + + lines.extend( + [ + "## Diff Table", + "", + "| Case ID | Change Type | Targeted | Before Route | After Route | Before Failure | After Failure | No-Call Changed |", + "| --- | --- | --- | --- | --- | --- | --- | --- |", + ] + ) + if summary["diff_table"]: + for row in summary["diff_table"]: + lines.append( + "| " + + " | ".join( + markdown_cell(row[key]) + for key in ( + "case_id", + "change_type", + "targeted", + "before_route", + "after_route", + "before_failure", + "after_failure", + "no_call_changed", + ) + ) + + " |" + ) + else: + lines.append("| _none_ | _none_ | _none_ | _none_ | _none_ | _none_ | _none_ | _none_ |") + + lines.extend( + [ + "", + "## Machine Readable Summary", + "", + "```json", + json.dumps(machine, ensure_ascii=False, indent=2), + "```", + "", + ] + ) + return "\n".join(lines) + + +def markdown_cell(value: Any) -> str: + if value is None: + return "" + return str(value).replace("|", "\\|").replace("\n", " ") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/routing-behavior-diff-audit/tests/test_routing_behavior_diff_audit.py b/skills/routing-behavior-diff-audit/tests/test_routing_behavior_diff_audit.py new file mode 100644 index 0000000..7dc3eea --- /dev/null +++ b/skills/routing-behavior-diff-audit/tests/test_routing_behavior_diff_audit.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +import json +import pathlib +import shutil +import subprocess +import sys +import tempfile +import unittest + + +SKILL_ROOT = pathlib.Path(__file__).resolve().parents[1] +REPO_ROOT = pathlib.Path(__file__).resolve().parents[3] +SCRIPT = SKILL_ROOT / "scripts" / "routing_behavior_diff_audit.py" +TMP_ROOT = REPO_ROOT / "tmp" / "routing-behavior-diff-audit-tests" + + +def run_cli(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(SCRIPT), *args], + text=True, + capture_output=True, + check=False, + ) + + +def write_json(path: pathlib.Path, data: object) -> pathlib.Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + return path + + +def load_summary(output_dir: pathlib.Path) -> dict[str, object]: + return json.loads((output_dir / "routing-behavior-diff.json").read_text(encoding="utf-8")) + + +class RoutingBehaviorDiffAuditTest(unittest.TestCase): + def setUp(self) -> None: + TMP_ROOT.mkdir(parents=True, exist_ok=True) + self.tmp_dir = tempfile.TemporaryDirectory(dir=TMP_ROOT) + self.root = pathlib.Path(self.tmp_dir.name) + self.before = self.root / "before.json" + self.after = self.root / "after.json" + self.output_dir = self.root / "outputs" + + def tearDown(self) -> None: + self.tmp_dir.cleanup() + + @classmethod + def tearDownClass(cls) -> None: + if TMP_ROOT.exists(): + shutil.rmtree(TMP_ROOT) + + def run_audit(self, *extra_args: str) -> tuple[subprocess.CompletedProcess[str], dict[str, object], str]: + result = run_cli( + "--before-results", + str(self.before), + "--after-results", + str(self.after), + "--output-dir", + str(self.output_dir), + "--case-id-field", + "case_id", + "--route-field", + "route", + "--expected-route-field", + "expected_route", + *extra_args, + ) + summary = load_summary(self.output_dir) + markdown = (self.output_dir / "routing-behavior-diff.md").read_text(encoding="utf-8") + return result, summary, markdown + + def test_unchanged_full_set_reports_all_cases_unchanged(self) -> None: + rows = [ + {"case_id": "case-1", "route": "alpha", "expected_route": "alpha"}, + {"case_id": "case-2", "route": "beta", "expected_route": "beta"}, + ] + write_json(self.before, rows) + write_json(self.after, rows) + + result, summary, markdown = self.run_audit() + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(summary["total_cases_compared"], 2) + self.assertEqual(summary["unchanged_cases"], ["case-1", "case-2"]) + self.assertEqual(summary["changed_cases"], []) + self.assertEqual(summary["non_target_changes"], []) + self.assertEqual(summary["machine_readable_summary"]["status"], "PASS") + self.assertIn("case-1", markdown) + + def test_targeted_route_changes_are_classified_separately(self) -> None: + write_json(self.before, [{"case_id": "case-1", "route": "alpha", "expected_route": "beta"}]) + write_json(self.after, [{"case_id": "case-1", "route": "beta", "expected_route": "beta"}]) + + result, summary, _ = self.run_audit("--targeted-case", "case-1") + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(summary["changed_cases"], ["case-1"]) + self.assertEqual(summary["targeted_changes"], ["case-1"]) + self.assertEqual(summary["non_target_changes"], []) + self.assertEqual(summary["resolved_failures"], ["case-1"]) + + def test_non_target_collateral_changes_are_blocker_candidates(self) -> None: + write_json( + self.before, + [ + {"case_id": "case-1", "route": "alpha"}, + {"case_id": "case-2", "route": "beta"}, + ], + ) + write_json( + self.after, + [ + {"case_id": "case-1", "route": "alpha"}, + {"case_id": "case-2", "route": "gamma"}, + ], + ) + + result, summary, _ = self.run_audit("--targeted-case", "case-1") + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(summary["changed_cases"], ["case-2"]) + self.assertEqual(summary["targeted_changes"], []) + self.assertEqual(summary["non_target_changes"], ["case-2"]) + self.assertEqual(summary["machine_readable_summary"]["status"], "ATTENTION") + + def test_case_missing_in_before_is_reported(self) -> None: + write_json(self.before, [{"case_id": "case-1", "route": "alpha"}]) + write_json( + self.after, + [ + {"case_id": "case-1", "route": "alpha"}, + {"case_id": "case-new", "route": "delta"}, + ], + ) + + result, summary, _ = self.run_audit() + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(summary["missing_before_cases"], ["case-new"]) + self.assertEqual(summary["missing_after_cases"], []) + + def test_case_missing_in_after_is_reported(self) -> None: + write_json( + self.before, + [ + {"case_id": "case-1", "route": "alpha"}, + {"case_id": "case-removed", "route": "delta"}, + ], + ) + write_json(self.after, [{"case_id": "case-1", "route": "alpha"}]) + + result, summary, _ = self.run_audit() + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(summary["missing_before_cases"], []) + self.assertEqual(summary["missing_after_cases"], ["case-removed"]) + + def test_changed_no_call_status_is_visible_in_diff_table(self) -> None: + write_json(self.before, [{"case_id": "case-1", "route": "no_call"}]) + write_json(self.after, [{"case_id": "case-1", "route": "alpha"}]) + + result, summary, _ = self.run_audit() + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(summary["changed_cases"], ["case-1"]) + self.assertTrue(summary["diff_table"][0]["no_call_changed"]) + + def test_new_failures_and_resolved_failures_are_classified(self) -> None: + write_json( + self.before, + [ + {"case_id": "case-new-fail", "route": "alpha", "expected_route": "alpha"}, + {"case_id": "case-resolved", "route": "wrong", "expected_route": "beta"}, + ], + ) + write_json( + self.after, + [ + {"case_id": "case-new-fail", "route": "wrong", "expected_route": "alpha"}, + {"case_id": "case-resolved", "route": "beta", "expected_route": "beta"}, + ], + ) + + result, summary, _ = self.run_audit() + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(summary["new_failures"], ["case-new-fail"]) + self.assertEqual(summary["resolved_failures"], ["case-resolved"]) + + def test_utf8_paths_and_chinese_filenames_are_preserved(self) -> None: + before_dir = self.root / "之前" + after_dir = self.root / "之后" + self.before = before_dir / "路由结果.json" + self.after = after_dir / "路由结果.json" + write_json(self.before, [{"case_id": "案例一", "route": "旧路由"}]) + write_json(self.after, [{"case_id": "案例一", "route": "新路由"}]) + + result, summary, markdown = self.run_audit() + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(summary["changed_cases"], ["案例一"]) + self.assertEqual(summary["diff_table"][0]["before_route"], "旧路由") + self.assertEqual(summary["diff_table"][0]["after_route"], "新路由") + self.assertIn("案例一", markdown) + + +if __name__ == "__main__": + unittest.main()