505 lines
22 KiB
Python
505 lines
22 KiB
Python
import importlib.util
|
||
import json
|
||
import tempfile
|
||
import unittest
|
||
from pathlib import Path
|
||
|
||
|
||
def load_validator():
|
||
script_path = Path(__file__).resolve().parents[1] / "scripts" / "validate_model_library.py"
|
||
spec = importlib.util.spec_from_file_location("validate_model_library", script_path)
|
||
module = importlib.util.module_from_spec(spec)
|
||
spec.loader.exec_module(module)
|
||
return module
|
||
|
||
|
||
class ValidateModelLibraryTests(unittest.TestCase):
|
||
def write_json(self, root, relative_path, data):
|
||
path = root / relative_path
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
|
||
def test_valid_library_has_no_errors(self):
|
||
validator = load_validator()
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
root = Path(tmp)
|
||
self.write_minimal_library(root)
|
||
|
||
errors = validator.validate_library(root)
|
||
|
||
self.assertEqual(errors, [])
|
||
|
||
def test_missing_source_reference_is_reported(self):
|
||
validator = load_validator()
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
root = Path(tmp)
|
||
self.write_json(root, "sources/source_articles.json", {"source_articles": []})
|
||
self.write_json(root, "sources/source_excerpts.json", {"source_excerpts": []})
|
||
self.write_json(root, "models/qpi.model.json", {
|
||
"model_id": "qpi",
|
||
"model_name": "QPI 三元定性模型",
|
||
"model_type": "routing",
|
||
"pipeline_position": "problem_definition",
|
||
"one_sentence_definition": "用于判断输入缺的是数据、路径还是共识。",
|
||
"core_question": "当前输入到底是提问、难题还是课题？",
|
||
"core_mechanism": "扫描核心匮乏物并匹配干预范式。",
|
||
"source_articles": ["missing_article"],
|
||
"source_evidence": ["missing_excerpt"],
|
||
"input_types": ["ambiguous_situation"],
|
||
"output_types": ["classification"],
|
||
"call_when": ["需要先判断问题类型"],
|
||
"do_not_call_when": ["事实检索已经足够"],
|
||
"common_misuses": ["把所有复杂局面都升维为课题"],
|
||
"failure_modes": ["误判核心匮乏物"],
|
||
"selection_priority": 90,
|
||
"confidence_level": "medium",
|
||
"stability_profile": {"status": "draft", "main_risks": ["样本仍少"]},
|
||
"regression_status": "draft",
|
||
"productization_notes": ["v0.1 样板模型"]
|
||
})
|
||
self.write_json(root, "tests/regression_cases.json", {"regression_cases": []})
|
||
|
||
errors = validator.validate_library(root)
|
||
|
||
self.assertIn("models/qpi.model.json references unknown source article missing_article", errors)
|
||
self.assertIn("models/qpi.model.json references unknown source excerpt missing_excerpt", errors)
|
||
|
||
def write_minimal_library(self, root, model_overrides=None, index_overrides=None):
|
||
model = {
|
||
"model_id": "qpi",
|
||
"model_name": "QPI 三元定性模型",
|
||
"model_type": "routing_model",
|
||
"pipeline_position": "pre_analysis",
|
||
"one_sentence_definition": "用于判断输入缺的是数据、路径还是共识。",
|
||
"core_question": "当前输入到底是提问、难题还是课题？",
|
||
"core_mechanism": "扫描核心匮乏物并匹配干预范式。",
|
||
"status": "draft",
|
||
"source_articles": ["article_qpi_primary_001"],
|
||
"source_evidence": ["excerpt_qpi_001"],
|
||
"input_types": ["模糊局面"],
|
||
"output_types": ["分类"],
|
||
"call_when": ["需要先判断问题类型"],
|
||
"do_not_call_when": ["事实检索已经足够"],
|
||
"trigger_keywords": ["这是什么问题"],
|
||
"negative_triggers": ["只查事实"],
|
||
"related_models": [],
|
||
"conflicting_models": [],
|
||
"disciplinary_anchors": ["问题定义理论"],
|
||
"common_misuses": ["把所有复杂局面都升维为课题"],
|
||
"failure_modes": ["误判核心匮乏物"],
|
||
"selection_priority": 9,
|
||
"confidence_level": "medium",
|
||
"stability_profile": {
|
||
"stability_level": "B",
|
||
"needs_stabilization": True,
|
||
"main_risks": ["样本仍少"],
|
||
"reason": "结构清晰但边界案例不足。",
|
||
"next_stabilization_action": "补充混合问题测试。"
|
||
},
|
||
"regression_status": "pending",
|
||
"example_inputs": ["这到底是什么问题？"],
|
||
"example_outputs": ["偏 Question，因为缺事实。"],
|
||
"output_contract": ["输出分类和理由"],
|
||
"structured_output_contract": {
|
||
"classification_scope": ["subject_contextual", "multi_perspective", "insufficient_context", "no_call"],
|
||
"is_provisional": "boolean",
|
||
"subject_position": "string | unknown",
|
||
"scenario_context": "string | unknown",
|
||
"responsibility_scope": ["individual_learning", "team_execution", "cross_function_governance", "owner_governance", "unknown"],
|
||
"context_sufficiency": ["high", "medium", "low"],
|
||
"missing_context": "array<string>",
|
||
"problem_owner": "string | unknown",
|
||
"problem_source": "string | unknown",
|
||
"time_scale": ["short_term", "mid_term", "long_term", "mixed", "unknown"],
|
||
"scarcity_profile": {
|
||
"data_scarcity": ["high", "medium", "low", "unknown"],
|
||
"path_or_resource_scarcity": ["high", "medium", "low", "unknown"],
|
||
"consensus_or_order_scarcity": ["high", "medium", "low", "unknown"]
|
||
},
|
||
"dominant_scarcity": ["data", "path_resource", "consensus_order", "mixed", "unknown"],
|
||
"classification": ["question", "problem", "issue", "mixed", "no_call"],
|
||
"classification_confidence": ["high", "medium", "low"],
|
||
"evidence_gap": "array<string>",
|
||
"misclassification_risk": "array<string>",
|
||
"recommended_next_step": ["检索", "工程拆解", "共识协调", "补充上下文", "不调用"],
|
||
"next_model_candidates": "array<model_id>"
|
||
},
|
||
"productization_notes": "作为前置路由模型。",
|
||
"version": "0.1",
|
||
"last_updated": "2026-06-16"
|
||
}
|
||
if model_overrides:
|
||
for key, value in model_overrides.items():
|
||
if value == "__DELETE__":
|
||
model.pop(key, None)
|
||
else:
|
||
model[key] = value
|
||
|
||
index = {
|
||
"index_version": "0.1",
|
||
"last_updated": "2026-06-16",
|
||
"models": [{
|
||
"model_id": "qpi",
|
||
"model_name": "QPI 三元定性模型",
|
||
"model_type": "routing_model",
|
||
"pipeline_position": "pre_analysis",
|
||
"model_file": "models/qpi.model.json",
|
||
"card_file": "cards/qpi.md",
|
||
"source_article_count": 1,
|
||
"source_evidence_count": 1,
|
||
"regression_case_count": 15,
|
||
"stability_level": "B",
|
||
"regression_status": "pending",
|
||
"status": "draft"
|
||
}]
|
||
}
|
||
if index_overrides:
|
||
index["models"][0].update(index_overrides)
|
||
|
||
self.write_json(root, "sources/source_articles.json", {
|
||
"source_articles": [{
|
||
"source_id": "article_qpi_primary_001",
|
||
"title": "问题之锚：从混沌现实到认知秩序的重构",
|
||
"source_type": "original_article",
|
||
"related_models": ["qpi"],
|
||
"source_status": "representative"
|
||
}]
|
||
})
|
||
self.write_json(root, "sources/source_excerpts.json", {
|
||
"source_excerpts": [{
|
||
"excerpt_id": "excerpt_qpi_001",
|
||
"source_id": "article_qpi_primary_001",
|
||
"related_model_id": "qpi",
|
||
"excerpt_type": "definition",
|
||
"summary": "QPI 将问题区分为提问、难题、课题。",
|
||
"used_for": ["core_mechanism"],
|
||
"quote_status": "exact",
|
||
"source_location": "test fixture"
|
||
}]
|
||
})
|
||
self.write_json(root, "models/qpi.model.json", model)
|
||
self.write_json(root, "models/model_index.json", index)
|
||
(root / "cards").mkdir(parents=True, exist_ok=True)
|
||
(root / "cards" / "qpi.md").write_text("# QPI\n", encoding="utf-8")
|
||
(root / "cards" / "card_index.md").write_text(
|
||
"| Model ID | Card |\n| --- | --- |\n| qpi | `cards/qpi.md` |\n",
|
||
encoding="utf-8"
|
||
)
|
||
self.write_json(root, "tests/regression_cases.json", {
|
||
"regression_cases": [
|
||
{
|
||
"case_id": "case_qpi_positive_001",
|
||
"model_id": "qpi",
|
||
"case_type": "positive",
|
||
"input": "我不知道错误码是什么。",
|
||
"expected_behavior": "识别为提问。",
|
||
"failure_signal": "输出难题或课题。"
|
||
},
|
||
{
|
||
"case_id": "case_qpi_boundary_001",
|
||
"model_id": "qpi",
|
||
"case_type": "boundary",
|
||
"input": "既缺数据也缺路径。",
|
||
"expected_behavior": "标出混合状态。",
|
||
"failure_signal": "单标签武断判断。"
|
||
},
|
||
{
|
||
"case_id": "case_qpi_misuse_001",
|
||
"model_id": "qpi",
|
||
"case_type": "misuse",
|
||
"input": "大环境不好所以我不用跟进客户。",
|
||
"expected_behavior": "识别恶意升维。",
|
||
"failure_signal": "接受甩锅叙事。"
|
||
},
|
||
{
|
||
"case_id": "case_qpi_positive_002",
|
||
"model_id": "qpi",
|
||
"case_type": "positive",
|
||
"input": "目标明确但缺少实现路径。",
|
||
"expected_behavior": "识别为难题。",
|
||
"failure_signal": "输出提问或课题。"
|
||
},
|
||
{
|
||
"case_id": "case_qpi_boundary_002",
|
||
"model_id": "qpi",
|
||
"case_type": "boundary",
|
||
"input": "团队既缺共识也有明确技术瓶颈。",
|
||
"expected_behavior": "标出主导匮乏物待判定。",
|
||
"failure_signal": "忽略混合状态。"
|
||
},
|
||
{
|
||
"case_id": "case_qpi_no_call_001",
|
||
"model_id": "qpi",
|
||
"case_type": "no_call",
|
||
"input": "请只改错别字。",
|
||
"expected_behavior": "不调用 QPI。",
|
||
"failure_signal": "过度分析。",
|
||
"should_call_model": False
|
||
},
|
||
{
|
||
"case_id": "case_qpi_selector_gate_001",
|
||
"model_id": "qpi",
|
||
"case_type": "selector_gate",
|
||
"input": "快速查一下定义。",
|
||
"expected_behavior": "低分或不调用。",
|
||
"failure_signal": "误召回。",
|
||
"negative_expected_models": ["intellectual_archaeology"]
|
||
},
|
||
{
|
||
"case_id": "case_qpi_pipeline_001",
|
||
"model_id": "qpi",
|
||
"case_type": "pipeline",
|
||
"input": "这是问题定义阶段。",
|
||
"expected_behavior": "QPI 位于前置路由。",
|
||
"failure_signal": "跳过 QPI。",
|
||
"expected_primary_model": "qpi"
|
||
},
|
||
{
|
||
"case_id": "case_qpi_positive_003",
|
||
"model_id": "qpi",
|
||
"case_type": "positive",
|
||
"input": "只缺一份报告。",
|
||
"expected_behavior": "识别为提问。",
|
||
"failure_signal": "过度升维。"
|
||
},
|
||
{
|
||
"case_id": "case_qpi_positive_004",
|
||
"model_id": "qpi",
|
||
"case_type": "positive",
|
||
"input": "目标清楚但没资源。",
|
||
"expected_behavior": "识别为难题。",
|
||
"failure_signal": "误判。"
|
||
},
|
||
{
|
||
"case_id": "case_qpi_positive_005",
|
||
"model_id": "qpi",
|
||
"case_type": "positive",
|
||
"input": "组织没有共识。",
|
||
"expected_behavior": "识别为课题。",
|
||
"failure_signal": "误判。"
|
||
},
|
||
{
|
||
"case_id": "case_qpi_boundary_003",
|
||
"model_id": "qpi",
|
||
"case_type": "boundary",
|
||
"input": "缺数据也缺共识。",
|
||
"expected_behavior": "识别混合。",
|
||
"failure_signal": "单标签。"
|
||
},
|
||
{
|
||
"case_id": "case_qpi_misuse_002",
|
||
"model_id": "qpi",
|
||
"case_type": "misuse",
|
||
"input": "员工离职都是个人不忠诚。",
|
||
"expected_behavior": "识别暴力降维风险。",
|
||
"failure_signal": "接受降维叙事。"
|
||
},
|
||
{
|
||
"case_id": "case_qpi_no_call_002",
|
||
"model_id": "qpi",
|
||
"case_type": "no_call",
|
||
"input": "翻译这句话。",
|
||
"expected_behavior": "不调用。",
|
||
"failure_signal": "过度分析。"
|
||
},
|
||
{
|
||
"case_id": "case_qpi_pipeline_002",
|
||
"model_id": "qpi",
|
||
"case_type": "pipeline",
|
||
"input": "QPI 先判断，再决定后续模型。",
|
||
"expected_behavior": "保持前置路由。",
|
||
"failure_signal": "跳过路由。"
|
||
}
|
||
]
|
||
})
|
||
|
||
def test_missing_trigger_keywords_is_reported(self):
|
||
validator = load_validator()
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
root = Path(tmp)
|
||
self.write_minimal_library(root, {"trigger_keywords": "__DELETE__"})
|
||
|
||
errors = validator.validate_library(root)
|
||
|
||
self.assertIn("models/qpi.model.json missing required field trigger_keywords", errors)
|
||
|
||
def test_invalid_selection_priority_is_reported(self):
|
||
validator = load_validator()
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
root = Path(tmp)
|
||
self.write_minimal_library(root, {"selection_priority": 95})
|
||
|
||
errors = validator.validate_library(root)
|
||
|
||
self.assertIn("models/qpi.model.json field selection_priority must be <= 10", errors)
|
||
|
||
def test_missing_status_is_reported(self):
|
||
validator = load_validator()
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
root = Path(tmp)
|
||
self.write_minimal_library(root, {"status": "__DELETE__"})
|
||
|
||
errors = validator.validate_library(root)
|
||
|
||
self.assertIn("models/qpi.model.json missing required field status", errors)
|
||
|
||
def test_invalid_model_type_is_reported(self):
|
||
validator = load_validator()
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
root = Path(tmp)
|
||
self.write_minimal_library(root, {"model_type": "routing"})
|
||
|
||
errors = validator.validate_library(root)
|
||
|
||
self.assertIn("models/qpi.model.json field model_type has invalid value routing", errors)
|
||
|
||
def test_missing_stability_reason_is_reported(self):
|
||
validator = load_validator()
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
root = Path(tmp)
|
||
self.write_minimal_library(root, {
|
||
"stability_profile": {
|
||
"stability_level": "B",
|
||
"needs_stabilization": True,
|
||
"main_risks": ["样本仍少"],
|
||
"next_stabilization_action": "补充测试。"
|
||
}
|
||
})
|
||
|
||
errors = validator.validate_library(root)
|
||
|
||
self.assertIn("models/qpi.model.json stability_profile missing required field reason", errors)
|
||
|
||
def test_model_index_missing_file_is_reported(self):
|
||
validator = load_validator()
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
root = Path(tmp)
|
||
self.write_minimal_library(root, index_overrides={"model_file": "models/missing.model.json"})
|
||
|
||
errors = validator.validate_library(root)
|
||
|
||
self.assertIn("models/model_index.json references missing model file models/missing.model.json", errors)
|
||
|
||
def test_model_index_count_drift_is_reported(self):
|
||
validator = load_validator()
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
root = Path(tmp)
|
||
self.write_minimal_library(root, index_overrides={
|
||
"source_article_count": 2,
|
||
"source_evidence_count": 2,
|
||
"regression_case_count": 99
|
||
})
|
||
|
||
errors = validator.validate_library(root)
|
||
|
||
self.assertIn("models/model_index.json entry qpi source_article_count is 2, expected 1", errors)
|
||
self.assertIn("models/model_index.json entry qpi source_evidence_count is 2, expected 1", errors)
|
||
self.assertIn("models/model_index.json entry qpi regression_case_count is 99, expected 15", errors)
|
||
|
||
def test_model_index_state_drift_is_reported(self):
|
||
validator = load_validator()
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
root = Path(tmp)
|
||
self.write_minimal_library(root, index_overrides={
|
||
"stability_level": "A",
|
||
"regression_status": "passed",
|
||
"status": "stable"
|
||
})
|
||
|
||
errors = validator.validate_library(root)
|
||
|
||
self.assertIn("models/model_index.json entry qpi stability_level is A, expected B", errors)
|
||
self.assertIn("models/model_index.json entry qpi regression_status is passed, expected pending", errors)
|
||
self.assertIn("models/model_index.json entry qpi status is stable, expected draft", errors)
|
||
|
||
def test_regression_case_coverage_is_reported(self):
|
||
validator = load_validator()
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
root = Path(tmp)
|
||
self.write_minimal_library(root)
|
||
self.write_json(root, "tests/regression_cases.json", {
|
||
"regression_cases": [{
|
||
"case_id": "case_qpi_positive_001",
|
||
"model_id": "qpi",
|
||
"case_type": "positive",
|
||
"input": "我不知道错误码是什么。",
|
||
"expected_behavior": "识别为提问。",
|
||
"failure_signal": "输出难题或课题。"
|
||
}]
|
||
})
|
||
|
||
errors = validator.validate_library(root)
|
||
|
||
self.assertIn("model qpi has 1 regression cases, expected at least 15", errors)
|
||
self.assertIn("model qpi missing regression case type boundary", errors)
|
||
self.assertIn("model qpi missing regression case type misuse", errors)
|
||
self.assertIn("model qpi missing regression case type no_call", errors)
|
||
self.assertIn("model qpi missing regression case type pipeline", errors)
|
||
self.assertIn("model qpi missing regression case type selector_gate", errors)
|
||
|
||
def test_qpi_missing_structured_output_contract_field_is_reported(self):
|
||
validator = load_validator()
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
root = Path(tmp)
|
||
self.write_minimal_library(root)
|
||
model = json.loads((root / "models" / "qpi.model.json").read_text(encoding="utf-8"))
|
||
model["structured_output_contract"].pop("classification_scope")
|
||
self.write_json(root, "models/qpi.model.json", model)
|
||
|
||
errors = validator.validate_library(root)
|
||
|
||
self.assertIn(
|
||
"models/qpi.model.json model qpi structured_output_contract missing required output field classification_scope",
|
||
errors
|
||
)
|
||
|
||
def test_qpi_digest_deprecated_fields_are_reported(self):
|
||
validator = load_validator()
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
root = Path(tmp)
|
||
self.write_minimal_library(root)
|
||
self.write_json(root, "selector/qpi_case_digests.json", {
|
||
"cases": [{
|
||
"case_id": "qpi-test-001",
|
||
"classification_scope": "subject_contextual",
|
||
"mixed_or_multi_perspective": "not_mixed",
|
||
"misframing_risks": ["violent_reduction"]
|
||
}]
|
||
})
|
||
|
||
errors = validator.validate_library(root)
|
||
|
||
self.assertIn(
|
||
"qpi case digest qpi-test-001 uses deprecated field misframing_risks; use misclassification_risk",
|
||
errors
|
||
)
|
||
self.assertIn(
|
||
"qpi case digest qpi-test-001 uses deprecated field mixed_or_multi_perspective; use qpi_complexity_pattern",
|
||
errors
|
||
)
|
||
|
||
def test_qpi_digest_multi_perspective_requires_viewpoint_detail(self):
|
||
validator = load_validator()
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
root = Path(tmp)
|
||
self.write_minimal_library(root)
|
||
self.write_json(root, "selector/qpi_case_digests.json", {
|
||
"cases": [{
|
||
"case_id": "qpi-test-002",
|
||
"classification_scope": "multi_perspective",
|
||
"qpi_complexity_pattern": "inter_viewpoint_divergence",
|
||
"misclassification_risk": ["single_viewpoint_only"]
|
||
}]
|
||
})
|
||
|
||
errors = validator.validate_library(root)
|
||
|
||
self.assertIn(
|
||
"qpi case digest qpi-test-002 multi_perspective case requires classification_by_viewpoint or viewpoint_summary",
|
||
errors
|
||
)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|