91 lines
3.0 KiB
Python
91 lines
3.0 KiB
Python
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from run_selector_demo import recommend
|
|
|
|
|
|
def read_json(path):
|
|
return json.loads(Path(path).read_text(encoding="utf-8"))
|
|
|
|
|
|
def load_selector_cases(root):
|
|
cases = []
|
|
for path in sorted((root / "tests").glob("*.regression.json")):
|
|
data = read_json(path)
|
|
for case in data.get("regression_cases", []):
|
|
if case.get("case_type") in {"no_call", "selector_gate", "pipeline"}:
|
|
cases.append(case)
|
|
return cases
|
|
|
|
|
|
def evaluate_case(root, case):
|
|
result = recommend(root, case["input"])
|
|
selected_ids = [item["model_id"] for item in result["selected_models"]]
|
|
errors = []
|
|
|
|
expected_primary = case.get("expected_primary_model")
|
|
if expected_primary and expected_primary != "none":
|
|
if not selected_ids or selected_ids[0] != expected_primary:
|
|
errors.append(f"expected primary {expected_primary}, got {selected_ids[:1] or ['none']}")
|
|
|
|
if expected_primary == "none" and result["selected_models"]:
|
|
errors.append(f"expected no selected model, got {selected_ids}")
|
|
|
|
for model_id in case.get("negative_expected_models", []):
|
|
if model_id in selected_ids:
|
|
errors.append(f"negative model selected: {model_id}")
|
|
|
|
if case.get("should_call_model") is False and case.get("model_id") in selected_ids:
|
|
errors.append(f"case model should not be called: {case.get('model_id')}")
|
|
|
|
return {
|
|
"case_id": case["case_id"],
|
|
"case_type": case["case_type"],
|
|
"input": case["input"],
|
|
"selected_models": selected_ids,
|
|
"no_call": result["no_call"],
|
|
"errors": errors
|
|
}
|
|
|
|
|
|
def write_report(root, results):
|
|
report_path = root / "reports" / "selector_regression_report_v0.2.md"
|
|
failures = [result for result in results if result["errors"]]
|
|
lines = [
|
|
"# Selector Regression Report v0.2",
|
|
"",
|
|
f"Status: `{'PASS' if not failures else 'FAIL'}`",
|
|
"",
|
|
f"Cases checked: {len(results)}",
|
|
f"Failures: {len(failures)}",
|
|
"",
|
|
"## Cases",
|
|
""
|
|
]
|
|
for result in results:
|
|
status = "PASS" if not result["errors"] else "FAIL"
|
|
lines.append(f"- `{result['case_id']}`: {status}; selected={result['selected_models']}; no_call={result['no_call']}")
|
|
for error in result["errors"]:
|
|
lines.append(f" - {error}")
|
|
report_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
return report_path
|
|
|
|
|
|
def main():
|
|
root = Path(__file__).resolve().parents[1]
|
|
results = [evaluate_case(root, case) for case in load_selector_cases(root)]
|
|
report_path = write_report(root, results)
|
|
print(f"selector regression report written to {report_path}")
|
|
failures = [result for result in results if result["errors"]]
|
|
if failures:
|
|
for result in failures:
|
|
print(f"ERROR: {result['case_id']}: {'; '.join(result['errors'])}")
|
|
return 1
|
|
print("selector regression passed")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|