import importlib.util import json import tempfile import unittest from pathlib import Path def load_validator(): script_path = Path(__file__).resolve().parents[1] / "scripts" / "validate_model_library.py" spec = importlib.util.spec_from_file_location("validate_model_library", script_path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module class ValidateModelLibraryTests(unittest.TestCase): def write_json(self, root, relative_path, data): path = root / relative_path path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") def test_valid_library_has_no_errors(self): validator = load_validator() with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) self.write_minimal_library(root) errors = validator.validate_library(root) self.assertEqual(errors, []) def test_missing_source_reference_is_reported(self): validator = load_validator() with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) self.write_json(root, "sources/source_articles.json", {"source_articles": []}) self.write_json(root, "sources/source_excerpts.json", {"source_excerpts": []}) self.write_json(root, "models/qpi.model.json", { "model_id": "qpi", "model_name": "QPI 三元定性模型", "model_type": "routing", "pipeline_position": "problem_definition", "one_sentence_definition": "用于判断输入缺的是数据、路径还是共识。", "core_question": "当前输入到底是提问、难题还是课题？", "core_mechanism": "扫描核心匮乏物并匹配干预范式。", "source_articles": ["missing_article"], "source_evidence": ["missing_excerpt"], "input_types": ["ambiguous_situation"], "output_types": ["classification"], "call_when": ["需要先判断问题类型"], "do_not_call_when": ["事实检索已经足够"], "common_misuses": ["把所有复杂局面都升维为课题"], "failure_modes": ["误判核心匮乏物"], "selection_priority": 90, "confidence_level": "medium", "stability_profile": {"status": "draft", "main_risks": ["样本仍少"]}, "regression_status": "draft", "productization_notes": ["v0.1 样板模型"] }) self.write_json(root, "tests/regression_cases.json", {"regression_cases": []}) errors = validator.validate_library(root) self.assertIn("models/qpi.model.json references unknown source article missing_article", errors) self.assertIn("models/qpi.model.json references unknown source excerpt missing_excerpt", errors) def write_minimal_library(self, root, model_overrides=None, index_overrides=None): model = { "model_id": "qpi", "model_name": "QPI 三元定性模型", "model_type": "routing_model", "pipeline_position": "pre_analysis", "one_sentence_definition": "用于判断输入缺的是数据、路径还是共识。", "core_question": "当前输入到底是提问、难题还是课题？", "core_mechanism": "扫描核心匮乏物并匹配干预范式。", "status": "draft", "source_articles": ["article_qpi_primary_001"], "source_evidence": ["excerpt_qpi_001"], "input_types": ["模糊局面"], "output_types": ["分类"], "call_when": ["需要先判断问题类型"], "do_not_call_when": ["事实检索已经足够"], "trigger_keywords": ["这是什么问题"], "negative_triggers": ["只查事实"], "related_models": [], "conflicting_models": [], "disciplinary_anchors": ["问题定义理论"], "common_misuses": ["把所有复杂局面都升维为课题"], "failure_modes": ["误判核心匮乏物"], "selection_priority": 9, "confidence_level": "medium", "stability_profile": { "stability_level": "B", "needs_stabilization": True, "main_risks": ["样本仍少"], "reason": "结构清晰但边界案例不足。", "next_stabilization_action": "补充混合问题测试。" }, "regression_status": "pending", "example_inputs": ["这到底是什么问题？"], "example_outputs": ["偏 Question，因为缺事实。"], "output_contract": ["输出分类和理由"], "structured_output_contract": { "classification_scope": ["subject_contextual", "multi_perspective", "insufficient_context", "no_call"], "is_provisional": "boolean", "subject_position": "string | unknown", "scenario_context": "string | unknown", "responsibility_scope": ["individual_learning", "team_execution", "cross_function_governance", "owner_governance", "unknown"], "context_sufficiency": ["high", "medium", "low"], "missing_context": "array", "problem_owner": "string | unknown", "problem_source": "string | unknown", "time_scale": ["short_term", "mid_term", "long_term", "mixed", "unknown"], "scarcity_profile": { "data_scarcity": ["high", "medium", "low", "unknown"], "path_or_resource_scarcity": ["high", "medium", "low", "unknown"], "consensus_or_order_scarcity": ["high", "medium", "low", "unknown"] }, "dominant_scarcity": ["data", "path_resource", "consensus_order", "mixed", "unknown"], "classification": ["question", "problem", "issue", "mixed", "no_call"], "classification_confidence": ["high", "medium", "low"], "evidence_gap": "array", "misclassification_risk": "array", "recommended_next_step": ["检索", "工程拆解", "共识协调", "补充上下文", "不调用"], "next_model_candidates": "array" }, "productization_notes": "作为前置路由模型。", "version": "0.1", "last_updated": "2026-06-16" } if model_overrides: for key, value in model_overrides.items(): if value == "__DELETE__": model.pop(key, None) else: model[key] = value index = { "index_version": "0.1", "last_updated": "2026-06-16", "models": [{ "model_id": "qpi", "model_name": "QPI 三元定性模型", "model_type": "routing_model", "pipeline_position": "pre_analysis", "model_file": "models/qpi.model.json", "card_file": "cards/qpi.md", "source_article_count": 1, "source_evidence_count": 1, "regression_case_count": 15, "stability_level": "B", "regression_status": "pending", "status": "draft" }] } if index_overrides: index["models"][0].update(index_overrides) self.write_json(root, "sources/source_articles.json", { "source_articles": [{ "source_id": "article_qpi_primary_001", "title": "问题之锚：从混沌现实到认知秩序的重构", "source_type": "original_article", "related_models": ["qpi"], "source_status": "representative" }] }) self.write_json(root, "sources/source_excerpts.json", { "source_excerpts": [{ "excerpt_id": "excerpt_qpi_001", "source_id": "article_qpi_primary_001", "related_model_id": "qpi", "excerpt_type": "definition", "summary": "QPI 将问题区分为提问、难题、课题。", "used_for": ["core_mechanism"], "quote_status": "exact", "source_location": "test fixture" }] }) self.write_json(root, "models/qpi.model.json", model) self.write_json(root, "models/model_index.json", index) (root / "cards").mkdir(parents=True, exist_ok=True) (root / "cards" / "qpi.md").write_text("# QPI\n", encoding="utf-8") (root / "cards" / "card_index.md").write_text( "| Model ID | Card |\n| --- | --- |\n| qpi | `cards/qpi.md` |\n", encoding="utf-8" ) self.write_json(root, "tests/regression_cases.json", { "regression_cases": [ { "case_id": "case_qpi_positive_001", "model_id": "qpi", "case_type": "positive", "input": "我不知道错误码是什么。", "expected_behavior": "识别为提问。", "failure_signal": "输出难题或课题。" }, { "case_id": "case_qpi_boundary_001", "model_id": "qpi", "case_type": "boundary", "input": "既缺数据也缺路径。", "expected_behavior": "标出混合状态。", "failure_signal": "单标签武断判断。" }, { "case_id": "case_qpi_misuse_001", "model_id": "qpi", "case_type": "misuse", "input": "大环境不好所以我不用跟进客户。", "expected_behavior": "识别恶意升维。", "failure_signal": "接受甩锅叙事。" }, { "case_id": "case_qpi_positive_002", "model_id": "qpi", "case_type": "positive", "input": "目标明确但缺少实现路径。", "expected_behavior": "识别为难题。", "failure_signal": "输出提问或课题。" }, { "case_id": "case_qpi_boundary_002", "model_id": "qpi", "case_type": "boundary", "input": "团队既缺共识也有明确技术瓶颈。", "expected_behavior": "标出主导匮乏物待判定。", "failure_signal": "忽略混合状态。" }, { "case_id": "case_qpi_no_call_001", "model_id": "qpi", "case_type": "no_call", "input": "请只改错别字。", "expected_behavior": "不调用 QPI。", "failure_signal": "过度分析。", "should_call_model": False }, { "case_id": "case_qpi_selector_gate_001", "model_id": "qpi", "case_type": "selector_gate", "input": "快速查一下定义。", "expected_behavior": "低分或不调用。", "failure_signal": "误召回。", "negative_expected_models": ["intellectual_archaeology"] }, { "case_id": "case_qpi_pipeline_001", "model_id": "qpi", "case_type": "pipeline", "input": "这是问题定义阶段。", "expected_behavior": "QPI 位于前置路由。", "failure_signal": "跳过 QPI。", "expected_primary_model": "qpi" }, { "case_id": "case_qpi_positive_003", "model_id": "qpi", "case_type": "positive", "input": "只缺一份报告。", "expected_behavior": "识别为提问。", "failure_signal": "过度升维。" }, { "case_id": "case_qpi_positive_004", "model_id": "qpi", "case_type": "positive", "input": "目标清楚但没资源。", "expected_behavior": "识别为难题。", "failure_signal": "误判。" }, { "case_id": "case_qpi_positive_005", "model_id": "qpi", "case_type": "positive", "input": "组织没有共识。", "expected_behavior": "识别为课题。", "failure_signal": "误判。" }, { "case_id": "case_qpi_boundary_003", "model_id": "qpi", "case_type": "boundary", "input": "缺数据也缺共识。", "expected_behavior": "识别混合。", "failure_signal": "单标签。" }, { "case_id": "case_qpi_misuse_002", "model_id": "qpi", "case_type": "misuse", "input": "员工离职都是个人不忠诚。", "expected_behavior": "识别暴力降维风险。", "failure_signal": "接受降维叙事。" }, { "case_id": "case_qpi_no_call_002", "model_id": "qpi", "case_type": "no_call", "input": "翻译这句话。", "expected_behavior": "不调用。", "failure_signal": "过度分析。" }, { "case_id": "case_qpi_pipeline_002", "model_id": "qpi", "case_type": "pipeline", "input": "QPI 先判断，再决定后续模型。", "expected_behavior": "保持前置路由。", "failure_signal": "跳过路由。" } ] }) def test_missing_trigger_keywords_is_reported(self): validator = load_validator() with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) self.write_minimal_library(root, {"trigger_keywords": "__DELETE__"}) errors = validator.validate_library(root) self.assertIn("models/qpi.model.json missing required field trigger_keywords", errors) def test_invalid_selection_priority_is_reported(self): validator = load_validator() with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) self.write_minimal_library(root, {"selection_priority": 95}) errors = validator.validate_library(root) self.assertIn("models/qpi.model.json field selection_priority must be <= 10", errors) def test_missing_status_is_reported(self): validator = load_validator() with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) self.write_minimal_library(root, {"status": "__DELETE__"}) errors = validator.validate_library(root) self.assertIn("models/qpi.model.json missing required field status", errors) def test_invalid_model_type_is_reported(self): validator = load_validator() with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) self.write_minimal_library(root, {"model_type": "routing"}) errors = validator.validate_library(root) self.assertIn("models/qpi.model.json field model_type has invalid value routing", errors) def test_missing_stability_reason_is_reported(self): validator = load_validator() with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) self.write_minimal_library(root, { "stability_profile": { "stability_level": "B", "needs_stabilization": True, "main_risks": ["样本仍少"], "next_stabilization_action": "补充测试。" } }) errors = validator.validate_library(root) self.assertIn("models/qpi.model.json stability_profile missing required field reason", errors) def test_model_index_missing_file_is_reported(self): validator = load_validator() with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) self.write_minimal_library(root, index_overrides={"model_file": "models/missing.model.json"}) errors = validator.validate_library(root) self.assertIn("models/model_index.json references missing model file models/missing.model.json", errors) def test_model_index_count_drift_is_reported(self): validator = load_validator() with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) self.write_minimal_library(root, index_overrides={ "source_article_count": 2, "source_evidence_count": 2, "regression_case_count": 99 }) errors = validator.validate_library(root) self.assertIn("models/model_index.json entry qpi source_article_count is 2, expected 1", errors) self.assertIn("models/model_index.json entry qpi source_evidence_count is 2, expected 1", errors) self.assertIn("models/model_index.json entry qpi regression_case_count is 99, expected 15", errors) def test_model_index_state_drift_is_reported(self): validator = load_validator() with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) self.write_minimal_library(root, index_overrides={ "stability_level": "A", "regression_status": "passed", "status": "stable" }) errors = validator.validate_library(root) self.assertIn("models/model_index.json entry qpi stability_level is A, expected B", errors) self.assertIn("models/model_index.json entry qpi regression_status is passed, expected pending", errors) self.assertIn("models/model_index.json entry qpi status is stable, expected draft", errors) def test_regression_case_coverage_is_reported(self): validator = load_validator() with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) self.write_minimal_library(root) self.write_json(root, "tests/regression_cases.json", { "regression_cases": [{ "case_id": "case_qpi_positive_001", "model_id": "qpi", "case_type": "positive", "input": "我不知道错误码是什么。", "expected_behavior": "识别为提问。", "failure_signal": "输出难题或课题。" }] }) errors = validator.validate_library(root) self.assertIn("model qpi has 1 regression cases, expected at least 15", errors) self.assertIn("model qpi missing regression case type boundary", errors) self.assertIn("model qpi missing regression case type misuse", errors) self.assertIn("model qpi missing regression case type no_call", errors) self.assertIn("model qpi missing regression case type pipeline", errors) self.assertIn("model qpi missing regression case type selector_gate", errors) def test_qpi_missing_structured_output_contract_field_is_reported(self): validator = load_validator() with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) self.write_minimal_library(root) model = json.loads((root / "models" / "qpi.model.json").read_text(encoding="utf-8")) model["structured_output_contract"].pop("classification_scope") self.write_json(root, "models/qpi.model.json", model) errors = validator.validate_library(root) self.assertIn( "models/qpi.model.json model qpi structured_output_contract missing required output field classification_scope", errors ) def test_qpi_digest_deprecated_fields_are_reported(self): validator = load_validator() with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) self.write_minimal_library(root) self.write_json(root, "selector/qpi_case_digests.json", { "cases": [{ "case_id": "qpi-test-001", "classification_scope": "subject_contextual", "mixed_or_multi_perspective": "not_mixed", "misframing_risks": ["violent_reduction"] }] }) errors = validator.validate_library(root) self.assertIn( "qpi case digest qpi-test-001 uses deprecated field misframing_risks; use misclassification_risk", errors ) self.assertIn( "qpi case digest qpi-test-001 uses deprecated field mixed_or_multi_perspective; use qpi_complexity_pattern", errors ) def test_qpi_digest_multi_perspective_requires_viewpoint_detail(self): validator = load_validator() with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) self.write_minimal_library(root) self.write_json(root, "selector/qpi_case_digests.json", { "cases": [{ "case_id": "qpi-test-002", "classification_scope": "multi_perspective", "qpi_complexity_pattern": "inter_viewpoint_divergence", "misclassification_risk": ["single_viewpoint_only"] }] }) errors = validator.validate_library(root) self.assertIn( "qpi case digest qpi-test-002 multi_perspective case requires classification_by_viewpoint or viewpoint_summary", errors ) if __name__ == "__main__": unittest.main()