feat: add routing behavior diff audit skill
This commit is contained in:
parent
c2a46b60ac
commit
7e19041048
|
|
@ -27,6 +27,7 @@ powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill repa
|
|||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill regression-validation-gate-runner
|
||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill review-bundle-audit
|
||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill review-context-builder
|
||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill routing-behavior-diff-audit
|
||||
```
|
||||
|
||||
Use `-Force` to back up and replace an existing installed copy:
|
||||
|
|
@ -38,6 +39,7 @@ powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill repa
|
|||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill regression-validation-gate-runner -Force
|
||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill review-bundle-audit -Force
|
||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill review-context-builder -Force
|
||||
powershell -ExecutionPolicy Bypass -File .\scripts\install-skill.ps1 -Skill routing-behavior-diff-audit -Force
|
||||
```
|
||||
|
||||
`-All` installs only registry rows with an installable status and `default` install policy. Optional Skills must be named explicitly.
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ Existing CCPE content is not migrated here.
|
|||
| `regression-validation-gate-runner` | Dry-run or execute project-declared regression and validation gates from a manifest while capturing logs, exit codes, durations, skipped gates, and changed-file notes. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\regression-validation-gate-runner` | `installed` | `default` | `C:\Users\wangq\.agents\skills\regression-validation-gate-runner` | none |
|
||||
| `review-bundle-audit` | Preflight review, handoff, release, CCRA, Web upload, or code-review bundle directories for required files, manifests, sidecars, reports, zip readability, warnings, and blockers. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\review-bundle-audit` | `installed` | `default` | `C:\Users\wangq\.agents\skills\review-bundle-audit` | none |
|
||||
| `review-context-builder` | Build file-first review, audit, planning, release, handoff, PR, or Agent invocation context indexes and manifests from configured local source roots, patterns, and metadata. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\review-context-builder` | `installed` | `default` | `C:\Users\wangq\.agents\skills\review-context-builder` | none |
|
||||
| `routing-behavior-diff-audit` | Compare before/after selector, classifier, or routing outputs for targeted changes, collateral non-target changes, no-call changes, missing cases, and expected-route mismatch changes. | `C:\Users\wangq\Documents\Codex\skills-vault\skills\routing-behavior-diff-audit` | `installed` | `default` | `C:\Users\wangq\.agents\skills\routing-behavior-diff-audit` | none |
|
||||
| `voice-generation` | Generate spoken audio from Markdown scripts using the local `voice-gen` CLI and MiniMax `mmx`, including custom voice clone registration and voice registry management. | `D:\AI\ClaudeCode\voice-ge` | `installed` | `conditional` | `C:\Users\wangq\.agents\skills\voice-generation` | none |
|
||||
|
||||
## Status Values
|
||||
|
|
|
|||
|
|
@ -0,0 +1,43 @@
|
|||
# Routing Behavior Diff Audit
|
||||
|
||||
Source Skill for deterministic before/after routing result comparison.
|
||||
|
||||
The canonical entry point is `SKILL.md`; the deterministic diff engine is in `scripts/routing_behavior_diff_audit.py`.
|
||||
|
||||
## Original Source
|
||||
|
||||
```text
|
||||
Original source: C:\Users\wangq\Documents\Codex\ccpe-system\requirements\skills-vault\2026-06-19-routing-behavior-diff-audit.md
|
||||
Migration date: 2026-06-19
|
||||
Migration status: first public Skill implementation
|
||||
Behavior preserved: deterministic routing-result diff only; no selector edits, expected-label updates, or patch-closure judgment
|
||||
Known gaps: Markdown support is limited to fenced JSON/YAML blocks; product-specific meaning remains outside this Skill
|
||||
```
|
||||
|
||||
## Layout
|
||||
|
||||
```text
|
||||
SKILL.md
|
||||
README.md
|
||||
agents/openai.yaml
|
||||
fixtures/before.example.json
|
||||
fixtures/after.example.json
|
||||
scripts/routing_behavior_diff_audit.py
|
||||
tests/test_routing_behavior_diff_audit.py
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```powershell
|
||||
conda run -n skills-vault python .\scripts\routing_behavior_diff_audit.py `
|
||||
--before-results C:\path\before.json `
|
||||
--after-results C:\path\after.json `
|
||||
--output-dir C:\path\helper-outputs `
|
||||
--targeted-case CASE-001
|
||||
```
|
||||
|
||||
## Tests
|
||||
|
||||
```powershell
|
||||
conda run -n skills-vault python -B -m unittest discover -s skills/routing-behavior-diff-audit/tests -v
|
||||
```
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
---
|
||||
name: routing-behavior-diff-audit
|
||||
description: Use when Codex needs to compare before/after selector, routing, QPI, classifier, rules-engine, triage, assignment, or model-governance outputs to identify targeted route changes, collateral non-target changes, no-call changes, missing cases, and new or resolved expected-route mismatches before review, release, handoff, CCRA/local review, or regression closeout.
|
||||
---
|
||||
|
||||
# Routing Behavior Diff Audit
|
||||
|
||||
## Purpose
|
||||
|
||||
Run a deterministic before/after behavior diff for rules-based routing outputs. Use the bundled script to compare case IDs, routes, optional expected routes, missing cases, no-call transitions, and targeted versus non-target changes.
|
||||
|
||||
This Skill reports observed differences only. It must not edit routing rules, change regression cases, update expected labels, infer product meaning, or decide whether a patch should close.
|
||||
|
||||
## Command
|
||||
|
||||
```powershell
|
||||
conda run -n skills-vault python .\scripts\routing_behavior_diff_audit.py `
|
||||
--before-results C:\path\before-results `
|
||||
--after-results C:\path\after-results `
|
||||
--output-dir C:\path\review-run `
|
||||
--case-id-field case_id `
|
||||
--route-field route `
|
||||
--expected-route-field expected_route `
|
||||
--targeted-case CASE-001
|
||||
```
|
||||
|
||||
When installed under `C:\Users\wangq\.agents\skills\routing-behavior-diff-audit`, run from that Skill directory or use the installed script path.
|
||||
|
||||
## Inputs
|
||||
|
||||
- `--before-results`: JSON, JSONL, or Markdown file, or a directory containing those files.
|
||||
- `--after-results`: JSON, JSONL, or Markdown file, or a directory containing those files.
|
||||
- `--output-dir`: directory where reports are written.
|
||||
- `--case-id-field`: field containing the stable case ID. Default: `case_id`.
|
||||
- `--route-field`: field containing the routing decision. Default: `route`.
|
||||
- `--expected-route-field`: optional field containing the expected route for failure comparison.
|
||||
- `--targeted-case`: repeatable case ID that the patch intentionally targets.
|
||||
- `--targeted-cases-file`: optional JSON/YAML list, or object with `targeted_cases`, `case_ids`, or `cases`.
|
||||
|
||||
JSON is the stable input target. Directory inputs are scanned recursively for `.json`, `.jsonl`, `.md`, and `.markdown`; Markdown support expects a fenced JSON or YAML block.
|
||||
|
||||
## Outputs
|
||||
|
||||
The script writes under `output_dir`:
|
||||
|
||||
```text
|
||||
routing-behavior-diff.md
|
||||
routing-behavior-diff.json
|
||||
```
|
||||
|
||||
The JSON report includes:
|
||||
|
||||
- `total_cases_compared`
|
||||
- `unchanged_cases`
|
||||
- `changed_cases`
|
||||
- `targeted_changes`
|
||||
- `non_target_changes`
|
||||
- `new_failures`
|
||||
- `resolved_failures`
|
||||
- `missing_before_cases`
|
||||
- `missing_after_cases`
|
||||
- `diff_table`
|
||||
- `machine_readable_summary`
|
||||
|
||||
`machine_readable_summary.status` is `PASS`, `ATTENTION`, or `FAIL` for automation routing only. It is not a patch approval, owner approval, CCRA decision, or release decision.
|
||||
|
||||
## Comparison Rules
|
||||
|
||||
- Compare only case IDs present in both before and after for `total_cases_compared`.
|
||||
- Classify after-only IDs as `missing_before_cases`.
|
||||
- Classify before-only IDs as `missing_after_cases`.
|
||||
- Treat a common case as changed when the route, expected route, no-call status, or expected-route failure status changes.
|
||||
- Classify changed cases listed by `--targeted-case` or `--targeted-cases-file` as `targeted_changes`.
|
||||
- Classify all other changed cases as `non_target_changes`.
|
||||
- Compute `new_failures` and `resolved_failures` only when `--expected-route-field` is supplied and present in the records.
|
||||
- Treat empty, null-like, and common no-call spellings such as `no_call`, `no-call`, and `no call` as no-call values.
|
||||
|
||||
## Safety
|
||||
|
||||
- Read only configured before/after result paths and optional targeted-cases file.
|
||||
- Write only `routing-behavior-diff.md` and `routing-behavior-diff.json` under `output_dir`.
|
||||
- Do not edit selectors, routing rules, regression cases, expected labels, review bundles, model cards, or governance artifacts.
|
||||
- Do not decide acceptance, lifecycle status, patch closure, release readiness, or owner approval.
|
||||
|
||||
## Validation
|
||||
|
||||
After changing diff behavior, run:
|
||||
|
||||
```powershell
|
||||
conda run -n skills-vault python -B -m unittest discover -s skills/routing-behavior-diff-audit/tests -v
|
||||
conda run -n skills-vault python -B C:\Users\wangq\.codex\skills\.system\skill-creator\scripts\quick_validate.py skills/routing-behavior-diff-audit
|
||||
```
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
interface:
|
||||
display_name: "Routing Behavior Diff Audit"
|
||||
short_description: "Compare before/after routing outputs"
|
||||
default_prompt: "Use $routing-behavior-diff-audit to compare before and after routing results and write routing-behavior-diff reports."
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
[
|
||||
{
|
||||
"case_id": "CASE-001",
|
||||
"route": "qpi-revised",
|
||||
"expected_route": "qpi-revised"
|
||||
},
|
||||
{
|
||||
"case_id": "CASE-002",
|
||||
"route": "review",
|
||||
"expected_route": "review"
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
[
|
||||
{
|
||||
"case_id": "CASE-001",
|
||||
"route": "qpi",
|
||||
"expected_route": "qpi"
|
||||
},
|
||||
{
|
||||
"case_id": "CASE-002",
|
||||
"route": "no_call",
|
||||
"expected_route": "review"
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,491 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
|
||||
RESULT_KEYS = ("results", "cases", "records", "items", "diff_table")
|
||||
SUPPORTED_SUFFIXES = {".json", ".jsonl", ".md", ".markdown"}
|
||||
NO_CALL_VALUES = {"", "none", "null", "no_call", "no-call", "no call", "nocall", "no decision"}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = parse_args(argv)
|
||||
targeted_cases, targeted_errors = load_targeted_cases(args.targeted_case, args.targeted_cases_file)
|
||||
before_cases, before_errors = load_case_map(
|
||||
args.before_results,
|
||||
args.case_id_field,
|
||||
args.route_field,
|
||||
args.expected_route_field,
|
||||
"before",
|
||||
)
|
||||
after_cases, after_errors = load_case_map(
|
||||
args.after_results,
|
||||
args.case_id_field,
|
||||
args.route_field,
|
||||
args.expected_route_field,
|
||||
"after",
|
||||
)
|
||||
blocking_errors = [*targeted_errors, *before_errors, *after_errors]
|
||||
|
||||
if blocking_errors:
|
||||
summary = empty_summary(blocking_errors)
|
||||
write_reports(args.output_dir, summary)
|
||||
print(f"routing-behavior-diff-audit: {len(blocking_errors)} blocking error(s)", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
summary = compare_cases(before_cases, after_cases, targeted_cases)
|
||||
write_reports(args.output_dir, summary)
|
||||
print(str(args.output_dir / "routing-behavior-diff.json"))
|
||||
return 0
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Compare before/after rules-based routing outputs and write deterministic diff reports.",
|
||||
)
|
||||
parser.add_argument("--before-results", required=True, type=pathlib.Path)
|
||||
parser.add_argument("--after-results", required=True, type=pathlib.Path)
|
||||
parser.add_argument("--output-dir", required=True, type=pathlib.Path)
|
||||
parser.add_argument("--case-id-field", default="case_id")
|
||||
parser.add_argument("--route-field", default="route")
|
||||
parser.add_argument("--expected-route-field", default=None)
|
||||
parser.add_argument("--targeted-case", action="append", default=[])
|
||||
parser.add_argument("--targeted-cases-file", type=pathlib.Path)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def load_targeted_cases(raw_cases: list[str], cases_file: pathlib.Path | None) -> tuple[set[str], list[str]]:
|
||||
targeted = {str(case_id) for case_id in raw_cases}
|
||||
errors: list[str] = []
|
||||
if not cases_file:
|
||||
return targeted, errors
|
||||
if not cases_file.exists():
|
||||
return targeted, [f"Targeted cases file not found: {cases_file}"]
|
||||
|
||||
try:
|
||||
loaded = parse_file(cases_file)
|
||||
except ValueError as exc:
|
||||
return targeted, [f"Unable to parse targeted cases file {cases_file}: {exc}"]
|
||||
|
||||
candidate: Any = loaded
|
||||
if isinstance(candidate, dict):
|
||||
for key in ("targeted_cases", "case_ids", "cases"):
|
||||
if key in candidate:
|
||||
candidate = candidate[key]
|
||||
break
|
||||
if not isinstance(candidate, list):
|
||||
return targeted, [f"Targeted cases file must contain a list or object with targeted_cases/case_ids/cases: {cases_file}"]
|
||||
|
||||
for item in candidate:
|
||||
if isinstance(item, dict):
|
||||
for key in ("case_id", "id"):
|
||||
if key in item:
|
||||
targeted.add(str(item[key]))
|
||||
break
|
||||
else:
|
||||
targeted.add(str(item))
|
||||
return targeted, errors
|
||||
|
||||
|
||||
def load_case_map(
|
||||
root: pathlib.Path,
|
||||
case_id_field: str,
|
||||
route_field: str,
|
||||
expected_route_field: str | None,
|
||||
side: str,
|
||||
) -> tuple[dict[str, dict[str, Any]], list[str]]:
|
||||
errors: list[str] = []
|
||||
if not root.exists():
|
||||
return {}, [f"{side} results path not found: {root}"]
|
||||
|
||||
files = collect_input_files(root)
|
||||
if not files:
|
||||
return {}, [f"{side} results path contains no supported files: {root}"]
|
||||
|
||||
cases: dict[str, dict[str, Any]] = {}
|
||||
for path in files:
|
||||
try:
|
||||
parsed = parse_file(path)
|
||||
except ValueError as exc:
|
||||
errors.append(f"Unable to parse {side} file {path}: {exc}")
|
||||
continue
|
||||
|
||||
records = extract_records(parsed)
|
||||
if not records:
|
||||
errors.append(f"No records found in {side} file: {path}")
|
||||
continue
|
||||
|
||||
for record in records:
|
||||
if not isinstance(record, dict):
|
||||
errors.append(f"Record in {side} file is not an object: {path}")
|
||||
continue
|
||||
if case_id_field not in record:
|
||||
errors.append(f"Record missing case id field '{case_id_field}' in {side} file: {path}")
|
||||
continue
|
||||
case_id = stable_string(record[case_id_field])
|
||||
if case_id in cases:
|
||||
errors.append(f"Duplicate case id '{case_id}' in {side} inputs")
|
||||
continue
|
||||
cases[case_id] = {
|
||||
"case_id": case_id,
|
||||
"route": normalize_value(record.get(route_field)),
|
||||
"expected_route": normalize_value(record.get(expected_route_field)) if expected_route_field else None,
|
||||
"source_file": str(path),
|
||||
}
|
||||
return cases, errors
|
||||
|
||||
|
||||
def collect_input_files(root: pathlib.Path) -> list[pathlib.Path]:
|
||||
if root.is_file():
|
||||
return [root] if root.suffix.lower() in SUPPORTED_SUFFIXES else []
|
||||
return sorted(path for path in root.rglob("*") if path.is_file() and path.suffix.lower() in SUPPORTED_SUFFIXES)
|
||||
|
||||
|
||||
def parse_file(path: pathlib.Path) -> Any:
|
||||
suffix = path.suffix.lower()
|
||||
text = path.read_text(encoding="utf-8")
|
||||
if suffix == ".json":
|
||||
return json.loads(text)
|
||||
if suffix == ".jsonl":
|
||||
records = []
|
||||
for line_number, line in enumerate(text.splitlines(), start=1):
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
try:
|
||||
records.append(json.loads(stripped))
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"invalid JSONL on line {line_number}: {exc}") from exc
|
||||
return records
|
||||
if suffix in {".md", ".markdown"}:
|
||||
return parse_markdown_summary(text, path)
|
||||
raise ValueError(f"unsupported suffix: {suffix}")
|
||||
|
||||
|
||||
def parse_markdown_summary(text: str, path: pathlib.Path) -> Any:
|
||||
fences = re.findall(r"```(?:json|yaml|yml)?\s*\n(.*?)\n```", text, flags=re.DOTALL | re.IGNORECASE)
|
||||
errors: list[str] = []
|
||||
for fence in fences:
|
||||
try:
|
||||
return json.loads(fence)
|
||||
except json.JSONDecodeError as exc:
|
||||
errors.append(f"json: {exc}")
|
||||
yaml_loaded = try_load_yaml(fence)
|
||||
if yaml_loaded is not None:
|
||||
return yaml_loaded
|
||||
raise ValueError(f"no parseable JSON/YAML fenced block found in Markdown summary {path}; {'; '.join(errors)}")
|
||||
|
||||
|
||||
def try_load_yaml(text: str) -> Any:
|
||||
try:
|
||||
import yaml # type: ignore[import-not-found]
|
||||
except ImportError:
|
||||
return None
|
||||
try:
|
||||
return yaml.safe_load(text)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def extract_records(parsed: Any) -> list[Any]:
|
||||
if isinstance(parsed, list):
|
||||
return parsed
|
||||
if isinstance(parsed, dict):
|
||||
for key in RESULT_KEYS:
|
||||
value = parsed.get(key)
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
if "case_id" in parsed or "id" in parsed:
|
||||
return [parsed]
|
||||
return []
|
||||
|
||||
|
||||
def compare_cases(before_cases: dict[str, dict[str, Any]], after_cases: dict[str, dict[str, Any]], targeted_cases: set[str]) -> dict[str, Any]:
|
||||
before_ids = set(before_cases)
|
||||
after_ids = set(after_cases)
|
||||
common_ids = sorted(before_ids & after_ids)
|
||||
missing_before = sorted(after_ids - before_ids)
|
||||
missing_after = sorted(before_ids - after_ids)
|
||||
|
||||
unchanged: list[str] = []
|
||||
changed: list[str] = []
|
||||
targeted_changes: list[str] = []
|
||||
non_target_changes: list[str] = []
|
||||
new_failures: list[str] = []
|
||||
resolved_failures: list[str] = []
|
||||
diff_table: list[dict[str, Any]] = []
|
||||
|
||||
for case_id in common_ids:
|
||||
before = before_cases[case_id]
|
||||
after = after_cases[case_id]
|
||||
before_failure = is_failure(before)
|
||||
after_failure = is_failure(after)
|
||||
route_changed = before["route"] != after["route"]
|
||||
expected_changed = before["expected_route"] != after["expected_route"]
|
||||
failure_status_changed = before_failure != after_failure
|
||||
no_call_changed = is_no_call(before["route"]) != is_no_call(after["route"])
|
||||
row_changed = route_changed or expected_changed or failure_status_changed or no_call_changed
|
||||
change_type = build_change_type(route_changed, expected_changed, failure_status_changed, no_call_changed)
|
||||
|
||||
if row_changed:
|
||||
changed.append(case_id)
|
||||
if case_id in targeted_cases:
|
||||
targeted_changes.append(case_id)
|
||||
else:
|
||||
non_target_changes.append(case_id)
|
||||
else:
|
||||
unchanged.append(case_id)
|
||||
|
||||
if not before_failure and after_failure:
|
||||
new_failures.append(case_id)
|
||||
if before_failure and not after_failure:
|
||||
resolved_failures.append(case_id)
|
||||
|
||||
diff_table.append(
|
||||
{
|
||||
"case_id": case_id,
|
||||
"change_type": change_type,
|
||||
"targeted": case_id in targeted_cases,
|
||||
"before_route": before["route"],
|
||||
"after_route": after["route"],
|
||||
"before_expected_route": before["expected_route"],
|
||||
"after_expected_route": after["expected_route"],
|
||||
"route_changed": route_changed,
|
||||
"expected_route_changed": expected_changed,
|
||||
"before_failure": before_failure,
|
||||
"after_failure": after_failure,
|
||||
"failure_status_changed": failure_status_changed,
|
||||
"no_call_changed": no_call_changed,
|
||||
"before_source_file": before["source_file"],
|
||||
"after_source_file": after["source_file"],
|
||||
}
|
||||
)
|
||||
|
||||
for case_id in missing_before:
|
||||
after = after_cases[case_id]
|
||||
diff_table.append(
|
||||
{
|
||||
"case_id": case_id,
|
||||
"change_type": "missing_before",
|
||||
"targeted": case_id in targeted_cases,
|
||||
"before_route": None,
|
||||
"after_route": after["route"],
|
||||
"before_expected_route": None,
|
||||
"after_expected_route": after["expected_route"],
|
||||
"route_changed": True,
|
||||
"expected_route_changed": after["expected_route"] is not None,
|
||||
"before_failure": None,
|
||||
"after_failure": is_failure(after),
|
||||
"failure_status_changed": None,
|
||||
"no_call_changed": None,
|
||||
"before_source_file": None,
|
||||
"after_source_file": after["source_file"],
|
||||
}
|
||||
)
|
||||
|
||||
for case_id in missing_after:
|
||||
before = before_cases[case_id]
|
||||
diff_table.append(
|
||||
{
|
||||
"case_id": case_id,
|
||||
"change_type": "missing_after",
|
||||
"targeted": case_id in targeted_cases,
|
||||
"before_route": before["route"],
|
||||
"after_route": None,
|
||||
"before_expected_route": before["expected_route"],
|
||||
"after_expected_route": None,
|
||||
"route_changed": True,
|
||||
"expected_route_changed": before["expected_route"] is not None,
|
||||
"before_failure": is_failure(before),
|
||||
"after_failure": None,
|
||||
"failure_status_changed": None,
|
||||
"no_call_changed": None,
|
||||
"before_source_file": before["source_file"],
|
||||
"after_source_file": None,
|
||||
}
|
||||
)
|
||||
|
||||
status = "ATTENTION" if non_target_changes or missing_before or missing_after or new_failures else "PASS"
|
||||
return {
|
||||
"total_cases_compared": len(common_ids),
|
||||
"unchanged_cases": unchanged,
|
||||
"changed_cases": changed,
|
||||
"targeted_changes": targeted_changes,
|
||||
"non_target_changes": non_target_changes,
|
||||
"new_failures": new_failures,
|
||||
"resolved_failures": resolved_failures,
|
||||
"missing_before_cases": missing_before,
|
||||
"missing_after_cases": missing_after,
|
||||
"diff_table": sorted(diff_table, key=lambda row: row["case_id"]),
|
||||
"blocking_errors": [],
|
||||
"machine_readable_summary": {
|
||||
"status": status,
|
||||
"total_cases_compared": len(common_ids),
|
||||
"changed_count": len(changed),
|
||||
"targeted_change_count": len(targeted_changes),
|
||||
"non_target_change_count": len(non_target_changes),
|
||||
"new_failure_count": len(new_failures),
|
||||
"resolved_failure_count": len(resolved_failures),
|
||||
"missing_before_count": len(missing_before),
|
||||
"missing_after_count": len(missing_after),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def build_change_type(route_changed: bool, expected_changed: bool, failure_status_changed: bool, no_call_changed: bool) -> str:
|
||||
labels: list[str] = []
|
||||
if route_changed:
|
||||
labels.append("route_changed")
|
||||
if expected_changed:
|
||||
labels.append("expected_route_changed")
|
||||
if failure_status_changed:
|
||||
labels.append("failure_status_changed")
|
||||
if no_call_changed:
|
||||
labels.append("no_call_changed")
|
||||
return "+".join(labels) if labels else "unchanged"
|
||||
|
||||
|
||||
def is_failure(case: dict[str, Any]) -> bool:
|
||||
expected = case.get("expected_route")
|
||||
if expected is None:
|
||||
return False
|
||||
return case.get("route") != expected
|
||||
|
||||
|
||||
def is_no_call(value: Any) -> bool:
|
||||
if value is None:
|
||||
return True
|
||||
return str(value).strip().casefold() in NO_CALL_VALUES
|
||||
|
||||
|
||||
def normalize_value(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
return stable_string(value)
|
||||
|
||||
|
||||
def stable_string(value: Any) -> str:
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return json.dumps(value, ensure_ascii=False, sort_keys=True)
|
||||
|
||||
|
||||
def empty_summary(blocking_errors: list[str]) -> dict[str, Any]:
|
||||
return {
|
||||
"total_cases_compared": 0,
|
||||
"unchanged_cases": [],
|
||||
"changed_cases": [],
|
||||
"targeted_changes": [],
|
||||
"non_target_changes": [],
|
||||
"new_failures": [],
|
||||
"resolved_failures": [],
|
||||
"missing_before_cases": [],
|
||||
"missing_after_cases": [],
|
||||
"diff_table": [],
|
||||
"blocking_errors": blocking_errors,
|
||||
"machine_readable_summary": {
|
||||
"status": "FAIL",
|
||||
"blocking_error_count": len(blocking_errors),
|
||||
"total_cases_compared": 0,
|
||||
"changed_count": 0,
|
||||
"targeted_change_count": 0,
|
||||
"non_target_change_count": 0,
|
||||
"new_failure_count": 0,
|
||||
"resolved_failure_count": 0,
|
||||
"missing_before_count": 0,
|
||||
"missing_after_count": 0,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def write_reports(output_dir: pathlib.Path, summary: dict[str, Any]) -> None:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
(output_dir / "routing-behavior-diff.json").write_text(
|
||||
json.dumps(summary, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(output_dir / "routing-behavior-diff.md").write_text(render_markdown(summary), encoding="utf-8")
|
||||
|
||||
|
||||
def render_markdown(summary: dict[str, Any]) -> str:
|
||||
machine = summary["machine_readable_summary"]
|
||||
lines = [
|
||||
"# Routing Behavior Diff",
|
||||
"",
|
||||
f"Status: `{machine['status']}`",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
f"- Total cases compared: {summary['total_cases_compared']}",
|
||||
f"- Unchanged cases: {len(summary['unchanged_cases'])}",
|
||||
f"- Changed cases: {len(summary['changed_cases'])}",
|
||||
f"- Targeted changes: {len(summary['targeted_changes'])}",
|
||||
f"- Non-target changes: {len(summary['non_target_changes'])}",
|
||||
f"- New failures: {len(summary['new_failures'])}",
|
||||
f"- Resolved failures: {len(summary['resolved_failures'])}",
|
||||
f"- Missing before cases: {len(summary['missing_before_cases'])}",
|
||||
f"- Missing after cases: {len(summary['missing_after_cases'])}",
|
||||
"",
|
||||
]
|
||||
if summary["blocking_errors"]:
|
||||
lines.extend(["## Blocking Errors", ""])
|
||||
lines.extend(f"- {error}" for error in summary["blocking_errors"])
|
||||
lines.append("")
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"## Diff Table",
|
||||
"",
|
||||
"| Case ID | Change Type | Targeted | Before Route | After Route | Before Failure | After Failure | No-Call Changed |",
|
||||
"| --- | --- | --- | --- | --- | --- | --- | --- |",
|
||||
]
|
||||
)
|
||||
if summary["diff_table"]:
|
||||
for row in summary["diff_table"]:
|
||||
lines.append(
|
||||
"| "
|
||||
+ " | ".join(
|
||||
markdown_cell(row[key])
|
||||
for key in (
|
||||
"case_id",
|
||||
"change_type",
|
||||
"targeted",
|
||||
"before_route",
|
||||
"after_route",
|
||||
"before_failure",
|
||||
"after_failure",
|
||||
"no_call_changed",
|
||||
)
|
||||
)
|
||||
+ " |"
|
||||
)
|
||||
else:
|
||||
lines.append("| _none_ | _none_ | _none_ | _none_ | _none_ | _none_ | _none_ | _none_ |")
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Machine Readable Summary",
|
||||
"",
|
||||
"```json",
|
||||
json.dumps(machine, ensure_ascii=False, indent=2),
|
||||
"```",
|
||||
"",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def markdown_cell(value: Any) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
return str(value).replace("|", "\\|").replace("\n", " ")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
|
@ -0,0 +1,210 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
SKILL_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]
|
||||
SCRIPT = SKILL_ROOT / "scripts" / "routing_behavior_diff_audit.py"
|
||||
TMP_ROOT = REPO_ROOT / "tmp" / "routing-behavior-diff-audit-tests"
|
||||
|
||||
|
||||
def run_cli(*args: str) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
[sys.executable, str(SCRIPT), *args],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def write_json(path: pathlib.Path, data: object) -> pathlib.Path:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def load_summary(output_dir: pathlib.Path) -> dict[str, object]:
|
||||
return json.loads((output_dir / "routing-behavior-diff.json").read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
class RoutingBehaviorDiffAuditTest(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
TMP_ROOT.mkdir(parents=True, exist_ok=True)
|
||||
self.tmp_dir = tempfile.TemporaryDirectory(dir=TMP_ROOT)
|
||||
self.root = pathlib.Path(self.tmp_dir.name)
|
||||
self.before = self.root / "before.json"
|
||||
self.after = self.root / "after.json"
|
||||
self.output_dir = self.root / "outputs"
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.tmp_dir.cleanup()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
if TMP_ROOT.exists():
|
||||
shutil.rmtree(TMP_ROOT)
|
||||
|
||||
def run_audit(self, *extra_args: str) -> tuple[subprocess.CompletedProcess[str], dict[str, object], str]:
|
||||
result = run_cli(
|
||||
"--before-results",
|
||||
str(self.before),
|
||||
"--after-results",
|
||||
str(self.after),
|
||||
"--output-dir",
|
||||
str(self.output_dir),
|
||||
"--case-id-field",
|
||||
"case_id",
|
||||
"--route-field",
|
||||
"route",
|
||||
"--expected-route-field",
|
||||
"expected_route",
|
||||
*extra_args,
|
||||
)
|
||||
summary = load_summary(self.output_dir)
|
||||
markdown = (self.output_dir / "routing-behavior-diff.md").read_text(encoding="utf-8")
|
||||
return result, summary, markdown
|
||||
|
||||
def test_unchanged_full_set_reports_all_cases_unchanged(self) -> None:
|
||||
rows = [
|
||||
{"case_id": "case-1", "route": "alpha", "expected_route": "alpha"},
|
||||
{"case_id": "case-2", "route": "beta", "expected_route": "beta"},
|
||||
]
|
||||
write_json(self.before, rows)
|
||||
write_json(self.after, rows)
|
||||
|
||||
result, summary, markdown = self.run_audit()
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(summary["total_cases_compared"], 2)
|
||||
self.assertEqual(summary["unchanged_cases"], ["case-1", "case-2"])
|
||||
self.assertEqual(summary["changed_cases"], [])
|
||||
self.assertEqual(summary["non_target_changes"], [])
|
||||
self.assertEqual(summary["machine_readable_summary"]["status"], "PASS")
|
||||
self.assertIn("case-1", markdown)
|
||||
|
||||
def test_targeted_route_changes_are_classified_separately(self) -> None:
|
||||
write_json(self.before, [{"case_id": "case-1", "route": "alpha", "expected_route": "beta"}])
|
||||
write_json(self.after, [{"case_id": "case-1", "route": "beta", "expected_route": "beta"}])
|
||||
|
||||
result, summary, _ = self.run_audit("--targeted-case", "case-1")
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(summary["changed_cases"], ["case-1"])
|
||||
self.assertEqual(summary["targeted_changes"], ["case-1"])
|
||||
self.assertEqual(summary["non_target_changes"], [])
|
||||
self.assertEqual(summary["resolved_failures"], ["case-1"])
|
||||
|
||||
def test_non_target_collateral_changes_are_blocker_candidates(self) -> None:
|
||||
write_json(
|
||||
self.before,
|
||||
[
|
||||
{"case_id": "case-1", "route": "alpha"},
|
||||
{"case_id": "case-2", "route": "beta"},
|
||||
],
|
||||
)
|
||||
write_json(
|
||||
self.after,
|
||||
[
|
||||
{"case_id": "case-1", "route": "alpha"},
|
||||
{"case_id": "case-2", "route": "gamma"},
|
||||
],
|
||||
)
|
||||
|
||||
result, summary, _ = self.run_audit("--targeted-case", "case-1")
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(summary["changed_cases"], ["case-2"])
|
||||
self.assertEqual(summary["targeted_changes"], [])
|
||||
self.assertEqual(summary["non_target_changes"], ["case-2"])
|
||||
self.assertEqual(summary["machine_readable_summary"]["status"], "ATTENTION")
|
||||
|
||||
def test_case_missing_in_before_is_reported(self) -> None:
|
||||
write_json(self.before, [{"case_id": "case-1", "route": "alpha"}])
|
||||
write_json(
|
||||
self.after,
|
||||
[
|
||||
{"case_id": "case-1", "route": "alpha"},
|
||||
{"case_id": "case-new", "route": "delta"},
|
||||
],
|
||||
)
|
||||
|
||||
result, summary, _ = self.run_audit()
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(summary["missing_before_cases"], ["case-new"])
|
||||
self.assertEqual(summary["missing_after_cases"], [])
|
||||
|
||||
def test_case_missing_in_after_is_reported(self) -> None:
|
||||
write_json(
|
||||
self.before,
|
||||
[
|
||||
{"case_id": "case-1", "route": "alpha"},
|
||||
{"case_id": "case-removed", "route": "delta"},
|
||||
],
|
||||
)
|
||||
write_json(self.after, [{"case_id": "case-1", "route": "alpha"}])
|
||||
|
||||
result, summary, _ = self.run_audit()
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(summary["missing_before_cases"], [])
|
||||
self.assertEqual(summary["missing_after_cases"], ["case-removed"])
|
||||
|
||||
def test_changed_no_call_status_is_visible_in_diff_table(self) -> None:
|
||||
write_json(self.before, [{"case_id": "case-1", "route": "no_call"}])
|
||||
write_json(self.after, [{"case_id": "case-1", "route": "alpha"}])
|
||||
|
||||
result, summary, _ = self.run_audit()
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(summary["changed_cases"], ["case-1"])
|
||||
self.assertTrue(summary["diff_table"][0]["no_call_changed"])
|
||||
|
||||
def test_new_failures_and_resolved_failures_are_classified(self) -> None:
|
||||
write_json(
|
||||
self.before,
|
||||
[
|
||||
{"case_id": "case-new-fail", "route": "alpha", "expected_route": "alpha"},
|
||||
{"case_id": "case-resolved", "route": "wrong", "expected_route": "beta"},
|
||||
],
|
||||
)
|
||||
write_json(
|
||||
self.after,
|
||||
[
|
||||
{"case_id": "case-new-fail", "route": "wrong", "expected_route": "alpha"},
|
||||
{"case_id": "case-resolved", "route": "beta", "expected_route": "beta"},
|
||||
],
|
||||
)
|
||||
|
||||
result, summary, _ = self.run_audit()
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(summary["new_failures"], ["case-new-fail"])
|
||||
self.assertEqual(summary["resolved_failures"], ["case-resolved"])
|
||||
|
||||
def test_utf8_paths_and_chinese_filenames_are_preserved(self) -> None:
|
||||
before_dir = self.root / "之前"
|
||||
after_dir = self.root / "之后"
|
||||
self.before = before_dir / "路由结果.json"
|
||||
self.after = after_dir / "路由结果.json"
|
||||
write_json(self.before, [{"case_id": "案例一", "route": "旧路由"}])
|
||||
write_json(self.after, [{"case_id": "案例一", "route": "新路由"}])
|
||||
|
||||
result, summary, markdown = self.run_audit()
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(summary["changed_cases"], ["案例一"])
|
||||
self.assertEqual(summary["diff_table"][0]["before_route"], "旧路由")
|
||||
self.assertEqual(summary["diff_table"][0]["after_route"], "新路由")
|
||||
self.assertIn("案例一", markdown)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Reference in New Issue