66 lines
2.6 KiB
Python
66 lines
2.6 KiB
Python
import importlib.util
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
|
|
def load_round04_runner():
|
|
script_path = Path(__file__).resolve().parents[1] / "scripts" / "run_round04_blind_routing.py"
|
|
spec = importlib.util.spec_from_file_location("run_round04_blind_routing", script_path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
class Round04BlindRoutingTests(unittest.TestCase):
|
|
def test_loads_frozen_blind_inputs_without_expected_routing(self):
|
|
runner = load_round04_runner()
|
|
root = Path(__file__).resolve().parents[1]
|
|
|
|
input_set = runner.load_blind_inputs(root)
|
|
|
|
self.assertEqual(input_set["blind_input_set_id"], "round04_blind_inputs_2026-06-18_v0.1")
|
|
self.assertEqual(len(input_set["inputs"]), 38)
|
|
for item in input_set["inputs"]:
|
|
self.assertIn("input_id", item)
|
|
self.assertIn("category_for_owner_review", item)
|
|
self.assertIn("input_text", item)
|
|
self.assertNotIn("expected_selector_behavior", item)
|
|
self.assertNotIn("expected_primary_model", item)
|
|
|
|
def test_evaluates_blind_input_without_answer_key_fields(self):
|
|
runner = load_round04_runner()
|
|
root = Path(__file__).resolve().parents[1]
|
|
input_set = runner.load_blind_inputs(root)
|
|
|
|
result = runner.evaluate_blind_input(root, input_set["inputs"][0])
|
|
|
|
self.assertEqual(result["input_id"], "R04-BI-001")
|
|
self.assertEqual(result["category_for_owner_review"], "explicit_no_call_fact_lookup")
|
|
self.assertIn("selected_models", result)
|
|
self.assertIn("rejected_models", result)
|
|
self.assertIn("no_call", result)
|
|
self.assertIn("routing_notes", result)
|
|
self.assertNotIn("expected_selector_behavior", result)
|
|
self.assertNotIn("expected_primary_model", result)
|
|
|
|
def test_report_states_results_are_not_scored_against_expected_routing(self):
|
|
runner = load_round04_runner()
|
|
root = Path(__file__).resolve().parents[1]
|
|
input_set = runner.load_blind_inputs(root)
|
|
results = [
|
|
runner.evaluate_blind_input(root, item)
|
|
for item in input_set["inputs"][:2]
|
|
]
|
|
|
|
report_text = runner.render_report(input_set, results)
|
|
|
|
self.assertIn("# Round 04 Blind Routing Evaluation Report", report_text)
|
|
self.assertIn("Expected routing labels: `not included before first blind run`", report_text)
|
|
self.assertIn("Inputs evaluated: 2", report_text)
|
|
self.assertIn("R04-BI-001", report_text)
|
|
self.assertIn("R04-BI-002", report_text)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|