import json
import sys
from pathlib import Path

from run_selector_demo import recommend


def read_json(path):
    return json.loads(Path(path).read_text(encoding="utf-8"))


def load_selector_cases(root):
    cases = []
    for path in sorted((root / "tests").glob("*.regression.json")):
        data = read_json(path)
        for case in data.get("regression_cases", []):
            if case.get("case_type") in {"no_call", "selector_gate", "pipeline"}:
                cases.append(case)
    return cases


def evaluate_case(root, case):
    result = recommend(root, case["input"])
    selected_ids = [item["model_id"] for item in result["selected_models"]]
    errors = []

    expected_primary = case.get("expected_primary_model")
    if expected_primary and expected_primary != "none":
        if not selected_ids or selected_ids[0] != expected_primary:
            errors.append(f"expected primary {expected_primary}, got {selected_ids[:1] or ['none']}")

    if expected_primary == "none" and result["selected_models"]:
        errors.append(f"expected no selected model, got {selected_ids}")

    for model_id in case.get("negative_expected_models", []):
        if model_id in selected_ids:
            errors.append(f"negative model selected: {model_id}")

    if case.get("should_call_model") is False and case.get("model_id") in selected_ids:
        errors.append(f"case model should not be called: {case.get('model_id')}")

    return {
        "case_id": case["case_id"],
        "case_type": case["case_type"],
        "input": case["input"],
        "selected_models": selected_ids,
        "no_call": result["no_call"],
        "errors": errors
    }


def write_report(root, results):
    report_path = root / "reports" / "selector_regression_report_v0.2.md"
    failures = [result for result in results if result["errors"]]
    lines = [
        "# Selector Regression Report v0.2",
        "",
        f"Status: `{'PASS' if not failures else 'FAIL'}`",
        "",
        f"Cases checked: {len(results)}",
        f"Failures: {len(failures)}",
        "",
        "## Cases",
        ""
    ]
    for result in results:
        status = "PASS" if not result["errors"] else "FAIL"
        lines.append(f"- `{result['case_id']}`: {status}; selected={result['selected_models']}; no_call={result['no_call']}")
        for error in result["errors"]:
            lines.append(f"  - {error}")
    report_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
    return report_path


def main():
    root = Path(__file__).resolve().parents[1]
    results = [evaluate_case(root, case) for case in load_selector_cases(root)]
    report_path = write_report(root, results)
    print(f"selector regression report written to {report_path}")
    failures = [result for result in results if result["errors"]]
    if failures:
        for result in failures:
            print(f"ERROR: {result['case_id']}: {'; '.join(result['errors'])}")
        return 1
    print("selector regression passed")
    return 0


if __name__ == "__main__":
    sys.exit(main())