import ast import re import sys from pathlib import Path SCRIPT_DIR = Path(__file__).resolve().parent if str(SCRIPT_DIR) not in sys.path: sys.path.insert(0, str(SCRIPT_DIR)) from run_round04_blind_routing import evaluate_blind_input, load_blind_inputs # noqa: E402 REPORT_NAME = "Round04_1_post_patch_routing_verification_report_2026-06-18.md" ORIGINAL_REPORT = "Round04_blind_routing_evaluation_report_2026-06-18.md" TARGET_EXPECTATIONS = { "R04-BI-002": { "expected_selected_models": [], "expected_no_call": True, "must_not_select": ["qpi", "intellectual_archaeology"], "review_label": "translation no-call; payload complexity must not trigger QPI", }, "R04-BI-024": { "expected_selected_models": ["qpi"], "expected_no_call": False, "must_not_select": ["intellectual_archaeology"], "review_label": "explicit IA refusal should hard-exclude IA while allowing QPI", }, "R04-BI-035": { "expected_selected_models": ["qpi"], "expected_no_call": False, "must_not_select": ["intellectual_archaeology"], "review_label": "depth-limiting plus dominant-scarcity judgment should allow QPI", }, "R04-BI-036": { "expected_selected_models": ["qpi"], "expected_no_call": False, "must_not_select": ["intellectual_archaeology"], "review_label": "depth-limiting plus scarcity-set judgment should allow QPI", }, } DIFF_CASE_NOTES = { "R04-BI-002": { "label": "target fix: translation no-call", "judgment": "Expected target change; translation instruction should not be routed to QPI because the payload contains `模型`.", }, "R04-BI-022": { "label": "accepted collateral change: depth-limited QPI override", "judgment": "Not a new failure. The request rejects deeper IA-style expansion while asking for dominant scarcity judgment, so lightweight QPI is acceptable and IA should remain rejected. Added to regression as a QPI selector-gate trap.", }, "R04-BI-024": { "label": "target fix: explicit IA refusal", "judgment": "Expected target change; explicit refusal should hard-exclude IA while allowing lightweight QPI.", }, "R04-BI-035": { "label": "target fix: dominant-scarcity depth limit", "judgment": "Expected target change; depth-limiting language should not block lightweight QPI scarcity judgment.", }, "R04-BI-036": { "label": "target fix: scarcity-set depth limit", "judgment": "Expected target change; depth-limiting language should not block QPI scarcity-set judgment.", }, } def parse_original_report(root): report_path = root / "reports" / ORIGINAL_REPORT text = report_path.read_text(encoding="utf-8") case_pattern = re.compile(r"^### (R04-BI-\d+)\n(?P.*?)(?=^### R04-BI-|\Z)", re.M | re.S) cases = {} for match in case_pattern.finditer(text): case_id = match.group(1) body = match.group("body") selected = extract_list_field(body, "selected_models") rejected = extract_list_field(body, "rejected_models") no_call = extract_bool_field(body, "no_call") cases[case_id] = { "selected_models": selected, "rejected_models": rejected, "no_call": no_call, } return cases def extract_list_field(body, field_name): match = re.search(rf"- {field_name}: `(?P.*?)`", body) if not match: return [] return ast.literal_eval(match.group("value")) def extract_bool_field(body, field_name): match = re.search(rf"- {field_name}: `(?PTrue|False)`", body) if not match: return None return match.group("value") == "True" def evaluate_expectation(result, expectation): errors = [] selected = result["selected_models"] for model_id in expectation["expected_selected_models"]: if model_id not in selected: errors.append(f"expected selected model missing: {model_id}") for model_id in expectation["must_not_select"]: if model_id in selected: errors.append(f"forbidden selected model: {model_id}") if result["no_call"] is not expectation["expected_no_call"]: errors.append(f"expected no_call={expectation['expected_no_call']}, got {result['no_call']}") if not expectation["expected_selected_models"] and selected: errors.append(f"expected no selected models, got {selected}") return errors def build_results(root): original = parse_original_report(root) input_set = load_blind_inputs(root) after_results = { item["input_id"]: evaluate_blind_input(root, item) for item in input_set["inputs"] } targeted = [] for case_id, expectation in TARGET_EXPECTATIONS.items(): after = after_results[case_id] errors = evaluate_expectation(after, expectation) targeted.append({ "case_id": case_id, "review_label": expectation["review_label"], "status": "PASS" if not errors else "FAIL", "errors": errors, "before": original.get(case_id, {}), "after": after, }) behavior_diffs = [] for case_id, before in original.items(): after = after_results.get(case_id) if not after: continue if ( before.get("selected_models") != after["selected_models"] or before.get("no_call") != after["no_call"] ): behavior_diffs.append({ "case_id": case_id, "before": before, "after": after, "note": DIFF_CASE_NOTES.get(case_id, { "label": "unclassified behavior change", "judgment": "Requires owner review.", }), }) return input_set, list(after_results.values()), targeted, behavior_diffs def build_current_post_patch_report(root): input_set, all_results, targeted, behavior_diffs = build_results(root) failures = [item for item in targeted if item["errors"]] no_call_count = sum(1 for item in all_results if item["no_call"]) lines = [ "# Round 04.1 Post-Patch Routing Verification Report", "", "Date: 2026-06-18", "", "This is not a second blind test.", "", f"Frozen input set: `{input_set['blind_input_set_id']}`", f"Original before-report: `reports/{ORIGINAL_REPORT}`", "", "## Scope", "", "- Round 04.1 verifies selector rule patches after owner / GPT review.", "- It reruns the frozen 38-input Round 04 pool for comparison.", "- Expected routing labels are limited to the four review-approved target cases.", "- It does not generate final answers and does not upgrade model status.", "", "## Summary", "", f"Frozen inputs rerun: {len(all_results)}", f"Post-patch no-call results: {no_call_count}", f"Targeted cases checked: {len(targeted)}", f"Targeted failures: {len(failures)}", f"Behavior changes recorded: {len(behavior_diffs)}", "", "## Targeted Case Results", "", ] for item in targeted: after = item["after"] before = item["before"] lines.extend([ f"### {item['case_id']}", "", f"- status: `{item['status']}`", f"- review_label: {item['review_label']}", f"- before_selected_models: `{before.get('selected_models')}`", f"- before_no_call: `{before.get('no_call')}`", f"- after_selected_models: `{after['selected_models']}`", f"- after_rejected_models: `{after['rejected_models']}`", f"- after_no_call: `{after['no_call']}`", ]) if item["errors"]: lines.append("- errors:") for error in item["errors"]: lines.append(f" - {error}") lines.extend([ "", "After selected model details:", ]) if after["selected_model_details"]: for detail in after["selected_model_details"]: lines.append(f"- `{detail['model_id']}` score={detail['score']} decision={detail['decision']}") if detail.get("reasons"): lines.append(" - reasons: " + "; ".join(detail["reasons"])) if detail.get("penalties"): lines.append(" - penalties: " + "; ".join(detail["penalties"])) else: lines.append("- none") lines.append("") lines.extend([ "## Non-target Behavior Diff / Accepted Collateral Change", "", "The frozen 38-input pool was rerun for comparison. Expected routing labels remain limited to the four target cases above, but the full rerun records five behavior changes from the original Round 04 report.", "", ]) for item in behavior_diffs: before = item["before"] after = item["after"] note = item["note"] lines.extend([ f"### {item['case_id']}", "", f"- label: {note['label']}", f"- before_selected_models: `{before.get('selected_models')}`", f"- before_no_call: `{before.get('no_call')}`", f"- after_selected_models: `{after['selected_models']}`", f"- after_rejected_models: `{after['rejected_models']}`", f"- after_no_call: `{after['no_call']}`", f"- closeout_judgment: {note['judgment']}", "", ]) lines.extend([ "## Full Pool Rerun Summary", "", ]) for result in all_results: lines.append( f"- `{result['input_id']}`: selected={result['selected_models']}; " f"no_call={result['no_call']}; category={result['category_for_owner_review']}" ) lines.append("") return "\n".join(lines) def build_post_patch_report(root): report_path = root / "reports" / REPORT_NAME if report_path.exists(): return report_path.read_text(encoding="utf-8") return build_current_post_patch_report(root) def write_report(root): report_path = root / "reports" / REPORT_NAME report_path.write_text(build_post_patch_report(root), encoding="utf-8") return report_path def main(): root = Path(__file__).resolve().parents[1] report_path = write_report(root) print(f"round04.1 post-patch verification report written to {report_path}") return 0 if __name__ == "__main__": sys.exit(main())