483 lines
20 KiB
Python
483 lines
20 KiB
Python
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
MODEL_REQUIRED_FIELDS = [
|
|
"model_id",
|
|
"model_name",
|
|
"model_type",
|
|
"pipeline_position",
|
|
"one_sentence_definition",
|
|
"core_question",
|
|
"core_mechanism",
|
|
"status",
|
|
"source_articles",
|
|
"source_evidence",
|
|
"input_types",
|
|
"output_types",
|
|
"call_when",
|
|
"do_not_call_when",
|
|
"trigger_keywords",
|
|
"negative_triggers",
|
|
"related_models",
|
|
"conflicting_models",
|
|
"disciplinary_anchors",
|
|
"common_misuses",
|
|
"failure_modes",
|
|
"selection_priority",
|
|
"confidence_level",
|
|
"stability_profile",
|
|
"regression_status",
|
|
"example_inputs",
|
|
"example_outputs",
|
|
"output_contract",
|
|
"productization_notes",
|
|
"version",
|
|
"last_updated",
|
|
]
|
|
|
|
MODEL_TYPE_VALUES = {
|
|
"routing_model",
|
|
"deep_modeling_model",
|
|
"lens_model",
|
|
"diagnostic_model",
|
|
"evaluation_model",
|
|
"generation_model",
|
|
"conflict_resolution_model",
|
|
"stabilization_model",
|
|
}
|
|
|
|
PIPELINE_POSITION_VALUES = {
|
|
"pre_analysis",
|
|
"analysis",
|
|
"deep_analysis",
|
|
"synthesis",
|
|
"red_team",
|
|
"evaluation",
|
|
"post_processing",
|
|
}
|
|
|
|
CONFIDENCE_LEVEL_VALUES = {"high", "medium", "low"}
|
|
REGRESSION_STATUS_VALUES = {"not_started", "pending", "in_progress", "passed", "failed", "needs_rebuild"}
|
|
STABILITY_LEVEL_VALUES = {"A", "B", "C", "D"}
|
|
REGRESSION_CASE_TYPE_VALUES = {"positive", "boundary", "misuse", "no_call", "selector_gate", "pipeline"}
|
|
REQUIRED_REGRESSION_CASE_TYPES = {"positive", "boundary", "misuse", "no_call", "selector_gate", "pipeline"}
|
|
MIN_REGRESSION_CASES_PER_MODEL = 15
|
|
EXPECTED_CLASSIFICATION_VALUES = {"question", "problem", "issue", "mixed", "no_call", "not_applicable"}
|
|
EXPECTED_DOMINANT_SCARCITY_VALUES = {"data", "path_resource", "consensus_order", "mixed", "unknown", "not_applicable"}
|
|
EXPECTED_MAX_DEPTH_VALUES = {"application", "domain", "process", "purpose", "core_mechanism", "human_capability", "philosophical_bedrock", "no_call", "not_applicable"}
|
|
EVALUATION_MODE_VALUES = {"manual", "keyword", "structured", "semantic"}
|
|
STATUS_VALUES = {"draft", "review", "reviewed", "callable", "stable", "archived", "deprecated", "draft_pre_contract"}
|
|
|
|
MODEL_SPECIFIC_STRUCTURED_OUTPUT_FIELDS = {
|
|
"qpi": [
|
|
"classification_scope",
|
|
"is_provisional",
|
|
"subject_position",
|
|
"scenario_context",
|
|
"responsibility_scope",
|
|
"context_sufficiency",
|
|
"missing_context",
|
|
"problem_owner",
|
|
"problem_source",
|
|
"time_scale",
|
|
"scarcity_profile",
|
|
"dominant_scarcity",
|
|
"classification",
|
|
"classification_confidence",
|
|
"evidence_gap",
|
|
"misclassification_risk",
|
|
"recommended_next_step",
|
|
"next_model_candidates",
|
|
],
|
|
"intellectual_archaeology": [
|
|
"should_call",
|
|
"entry_reason",
|
|
"recommended_max_depth",
|
|
"layers_to_analyze",
|
|
"stop_reason",
|
|
"no_deeper_reason",
|
|
"assumptions_by_layer",
|
|
"validation_needed",
|
|
"action_implication",
|
|
],
|
|
}
|
|
|
|
STABILITY_REQUIRED_FIELDS = [
|
|
"stability_level",
|
|
"needs_stabilization",
|
|
"main_risks",
|
|
"reason",
|
|
"next_stabilization_action",
|
|
]
|
|
|
|
SOURCE_ARTICLE_REQUIRED_FIELDS = [
|
|
"source_id",
|
|
"title",
|
|
"source_type",
|
|
"related_models",
|
|
"source_status",
|
|
]
|
|
|
|
SOURCE_EXCERPT_REQUIRED_FIELDS = [
|
|
"excerpt_id",
|
|
"source_id",
|
|
"related_model_id",
|
|
"excerpt_type",
|
|
"summary",
|
|
"used_for",
|
|
"quote_status",
|
|
"source_location",
|
|
]
|
|
|
|
REGRESSION_CASE_REQUIRED_FIELDS = [
|
|
"case_id",
|
|
"model_id",
|
|
"case_type",
|
|
"input",
|
|
"expected_behavior",
|
|
"failure_signal",
|
|
]
|
|
|
|
SOURCE_TYPE_VALUES = {"original_article", "synthesis_note", "placeholder"}
|
|
SOURCE_STATUS_VALUES = {"representative", "derived_synthesis", "placeholder"}
|
|
EXCERPT_TYPE_VALUES = {"definition", "taxonomy", "mechanism", "application_rule", "value_claim", "boundary_rule", "validation_rule"}
|
|
QUOTE_STATUS_VALUES = {"exact", "condensed", "paraphrased"}
|
|
|
|
|
|
def read_json(path):
|
|
if not path.exists():
|
|
return None, [f"missing file {path.as_posix()}"]
|
|
try:
|
|
return json.loads(path.read_text(encoding="utf-8")), []
|
|
except json.JSONDecodeError as exc:
|
|
return None, [f"{path.as_posix()} is invalid JSON: {exc}"]
|
|
|
|
|
|
def require_fields(record, fields, label):
|
|
return [f"{label} missing required field {field}" for field in fields if field not in record]
|
|
|
|
|
|
def collect_duplicate_errors(records, key, label):
|
|
seen = set()
|
|
errors = []
|
|
for record in records:
|
|
value = record.get(key)
|
|
if value in seen:
|
|
errors.append(f"duplicate {label} {value}")
|
|
seen.add(value)
|
|
return errors
|
|
|
|
|
|
def load_collection(root, relative_path, collection_key):
|
|
data, errors = read_json(root / relative_path)
|
|
if errors:
|
|
return [], errors
|
|
if not isinstance(data, dict) or collection_key not in data or not isinstance(data[collection_key], list):
|
|
return [], [f"{relative_path} must contain list field {collection_key}"]
|
|
return data[collection_key], []
|
|
|
|
|
|
def validate_enum(record, field, allowed_values, label):
|
|
value = record.get(field)
|
|
if value is not None and value not in allowed_values:
|
|
return [f"{label} field {field} has invalid value {value}"]
|
|
return []
|
|
|
|
|
|
def validate_integer_range(record, field, minimum, maximum, label):
|
|
value = record.get(field)
|
|
if value is None:
|
|
return []
|
|
errors = []
|
|
if not isinstance(value, int):
|
|
return [f"{label} field {field} must be an integer"]
|
|
if value < minimum:
|
|
errors.append(f"{label} field {field} must be >= {minimum}")
|
|
if value > maximum:
|
|
errors.append(f"{label} field {field} must be <= {maximum}")
|
|
return errors
|
|
|
|
|
|
def validate_model_contract(model, label):
|
|
errors = []
|
|
errors.extend(require_fields(model, MODEL_REQUIRED_FIELDS, label))
|
|
errors.extend(validate_enum(model, "model_type", MODEL_TYPE_VALUES, label))
|
|
errors.extend(validate_enum(model, "pipeline_position", PIPELINE_POSITION_VALUES, label))
|
|
errors.extend(validate_enum(model, "confidence_level", CONFIDENCE_LEVEL_VALUES, label))
|
|
errors.extend(validate_enum(model, "regression_status", REGRESSION_STATUS_VALUES, label))
|
|
errors.extend(validate_enum(model, "status", STATUS_VALUES, label))
|
|
errors.extend(validate_integer_range(model, "selection_priority", 1, 10, label))
|
|
|
|
stability_profile = model.get("stability_profile")
|
|
if isinstance(stability_profile, dict):
|
|
errors.extend(require_fields(stability_profile, STABILITY_REQUIRED_FIELDS, f"{label} stability_profile"))
|
|
errors.extend(validate_enum(stability_profile, "stability_level", STABILITY_LEVEL_VALUES, f"{label} stability_profile"))
|
|
needs_stabilization = stability_profile.get("needs_stabilization")
|
|
if needs_stabilization is not None and not isinstance(needs_stabilization, bool):
|
|
errors.append(f"{label} stability_profile field needs_stabilization must be a boolean")
|
|
elif stability_profile is not None:
|
|
errors.append(f"{label} field stability_profile must be an object")
|
|
|
|
errors.extend(validate_structured_output_contract(model, label))
|
|
|
|
return errors
|
|
|
|
|
|
def validate_structured_output_contract(model, label):
|
|
model_id = model.get("model_id")
|
|
required_fields = MODEL_SPECIFIC_STRUCTURED_OUTPUT_FIELDS.get(model_id, [])
|
|
if not required_fields:
|
|
return []
|
|
|
|
contract = model.get("structured_output_contract")
|
|
if not isinstance(contract, dict):
|
|
return [f"{label} model {model_id} missing structured_output_contract object"]
|
|
|
|
errors = []
|
|
for field in required_fields:
|
|
if field not in contract:
|
|
errors.append(f"{label} model {model_id} structured_output_contract missing required output field {field}")
|
|
return errors
|
|
|
|
|
|
def load_model_index(root):
|
|
path = root / "models" / "model_index.json"
|
|
data, errors = read_json(path)
|
|
if errors:
|
|
return None, errors
|
|
if not isinstance(data, dict) or "models" not in data or not isinstance(data["models"], list):
|
|
return None, ["models/model_index.json must contain list field models"]
|
|
return data, []
|
|
|
|
|
|
def card_index_text(root):
|
|
path = root / "cards" / "card_index.md"
|
|
if not path.exists():
|
|
return None, ["missing file cards/card_index.md"]
|
|
return path.read_text(encoding="utf-8"), []
|
|
|
|
|
|
def parse_card_index_model_ids(text):
|
|
model_ids = set()
|
|
for line in text.splitlines():
|
|
stripped = line.strip()
|
|
if not stripped.startswith("|") or stripped.startswith("| ---") or "Model ID" in stripped:
|
|
continue
|
|
cells = [cell.strip() for cell in stripped.strip("|").split("|")]
|
|
if cells and cells[0]:
|
|
model_ids.add(cells[0])
|
|
return model_ids
|
|
|
|
|
|
def validate_library(root):
|
|
root = Path(root)
|
|
errors = []
|
|
|
|
source_articles, article_errors = load_collection(root, "sources/source_articles.json", "source_articles")
|
|
source_excerpts, excerpt_errors = load_collection(root, "sources/source_excerpts.json", "source_excerpts")
|
|
regression_cases, case_errors = load_collection(root, "tests/regression_cases.json", "regression_cases")
|
|
errors.extend(article_errors)
|
|
errors.extend(excerpt_errors)
|
|
errors.extend(case_errors)
|
|
|
|
for article in source_articles:
|
|
errors.extend(require_fields(article, SOURCE_ARTICLE_REQUIRED_FIELDS, f"source article {article.get('source_id', '<missing>')}"))
|
|
errors.extend(validate_enum(article, "source_type", SOURCE_TYPE_VALUES, f"source article {article.get('source_id', '<missing>')}"))
|
|
errors.extend(validate_enum(article, "source_status", SOURCE_STATUS_VALUES, f"source article {article.get('source_id', '<missing>')}"))
|
|
for excerpt in source_excerpts:
|
|
errors.extend(require_fields(excerpt, SOURCE_EXCERPT_REQUIRED_FIELDS, f"source excerpt {excerpt.get('excerpt_id', '<missing>')}"))
|
|
errors.extend(validate_enum(excerpt, "excerpt_type", EXCERPT_TYPE_VALUES, f"source excerpt {excerpt.get('excerpt_id', '<missing>')}"))
|
|
errors.extend(validate_enum(excerpt, "quote_status", QUOTE_STATUS_VALUES, f"source excerpt {excerpt.get('excerpt_id', '<missing>')}"))
|
|
if excerpt.get("quote_status") == "exact":
|
|
raw_excerpt = excerpt.get("raw_excerpt", "")
|
|
if "..." in raw_excerpt or "……" in raw_excerpt:
|
|
errors.append(f"source excerpt {excerpt.get('excerpt_id')} quote_status exact must not contain ellipsis")
|
|
if excerpt.get("quote_status") == "condensed" and not excerpt.get("notes"):
|
|
errors.append(f"source excerpt {excerpt.get('excerpt_id')} quote_status condensed requires notes")
|
|
for case in regression_cases:
|
|
errors.extend(require_fields(case, REGRESSION_CASE_REQUIRED_FIELDS, f"regression case {case.get('case_id', '<missing>')}"))
|
|
errors.extend(validate_enum(case, "case_type", REGRESSION_CASE_TYPE_VALUES, f"regression case {case.get('case_id', '<missing>')}"))
|
|
errors.extend(validate_enum(case, "expected_classification", EXPECTED_CLASSIFICATION_VALUES, f"regression case {case.get('case_id', '<missing>')}"))
|
|
errors.extend(validate_enum(case, "expected_dominant_scarcity", EXPECTED_DOMINANT_SCARCITY_VALUES, f"regression case {case.get('case_id', '<missing>')}"))
|
|
errors.extend(validate_enum(case, "expected_max_depth", EXPECTED_MAX_DEPTH_VALUES, f"regression case {case.get('case_id', '<missing>')}"))
|
|
errors.extend(validate_enum(case, "evaluation_mode", EVALUATION_MODE_VALUES, f"regression case {case.get('case_id', '<missing>')}"))
|
|
if "should_call_model" in case and not isinstance(case.get("should_call_model"), bool):
|
|
errors.append(f"regression case {case.get('case_id')} field should_call_model must be a boolean")
|
|
|
|
errors.extend(collect_duplicate_errors(source_articles, "source_id", "source article id"))
|
|
errors.extend(collect_duplicate_errors(source_excerpts, "excerpt_id", "source excerpt id"))
|
|
errors.extend(collect_duplicate_errors(regression_cases, "case_id", "regression case id"))
|
|
|
|
article_ids = {article.get("source_id") for article in source_articles}
|
|
excerpt_ids = {excerpt.get("excerpt_id") for excerpt in source_excerpts}
|
|
|
|
model_records = []
|
|
model_paths = sorted((root / "models").glob("*.model.json")) if (root / "models").exists() else []
|
|
if not model_paths:
|
|
errors.append("models/ contains no *.model.json files")
|
|
|
|
for model_path in model_paths:
|
|
relative_model_path = model_path.relative_to(root).as_posix()
|
|
model, model_errors = read_json(model_path)
|
|
errors.extend(model_errors)
|
|
if model is None:
|
|
continue
|
|
model_records.append((relative_model_path, model))
|
|
errors.extend(validate_model_contract(model, relative_model_path))
|
|
|
|
model_ids = {model.get("model_id") for _, model in model_records}
|
|
models_by_id = {model.get("model_id"): (relative_path, model) for relative_path, model in model_records}
|
|
errors.extend(collect_duplicate_errors([model for _, model in model_records], "model_id", "model id"))
|
|
|
|
for relative_model_path, model in model_records:
|
|
for source_id in model.get("source_articles", []):
|
|
if source_id not in article_ids:
|
|
errors.append(f"{relative_model_path} references unknown source article {source_id}")
|
|
for excerpt_id in model.get("source_evidence", []):
|
|
if excerpt_id not in excerpt_ids:
|
|
errors.append(f"{relative_model_path} references unknown source excerpt {excerpt_id}")
|
|
|
|
for excerpt in source_excerpts:
|
|
source_id = excerpt.get("source_id")
|
|
related_model_id = excerpt.get("related_model_id")
|
|
if source_id and source_id not in article_ids:
|
|
errors.append(f"source excerpt {excerpt.get('excerpt_id')} references unknown source article {source_id}")
|
|
if related_model_id and related_model_id not in model_ids:
|
|
errors.append(f"source excerpt {excerpt.get('excerpt_id')} references unknown model {related_model_id}")
|
|
|
|
for case in regression_cases:
|
|
model_id = case.get("model_id")
|
|
if model_id and model_id not in model_ids:
|
|
errors.append(f"regression case {case.get('case_id')} references unknown model {model_id}")
|
|
|
|
model_index, model_index_errors = load_model_index(root)
|
|
errors.extend(model_index_errors)
|
|
indexed_model_ids = set()
|
|
if model_index:
|
|
for entry in model_index["models"]:
|
|
entry_id = entry.get("model_id")
|
|
indexed_model_ids.add(entry_id)
|
|
model_file = entry.get("model_file")
|
|
card_file = entry.get("card_file")
|
|
if model_file and not (root / model_file).exists():
|
|
errors.append(f"models/model_index.json references missing model file {model_file}")
|
|
if card_file and not (root / card_file).exists():
|
|
errors.append(f"models/model_index.json references missing card file {card_file}")
|
|
if entry_id in models_by_id:
|
|
relative_model_path, model = models_by_id[entry_id]
|
|
expected_values = {
|
|
"model_name": model.get("model_name"),
|
|
"model_type": model.get("model_type"),
|
|
"pipeline_position": model.get("pipeline_position"),
|
|
"model_file": relative_model_path,
|
|
"source_article_count": len(model.get("source_articles", [])),
|
|
"source_evidence_count": len(model.get("source_evidence", [])),
|
|
"stability_level": model.get("stability_profile", {}).get("stability_level") if isinstance(model.get("stability_profile"), dict) else None,
|
|
"regression_status": model.get("regression_status"),
|
|
"status": model.get("status"),
|
|
}
|
|
for field, expected_value in expected_values.items():
|
|
actual_value = entry.get(field)
|
|
if actual_value != expected_value:
|
|
errors.append(f"models/model_index.json entry {entry_id} {field} is {actual_value}, expected {expected_value}")
|
|
|
|
card_index, card_index_errors = card_index_text(root)
|
|
errors.extend(card_index_errors)
|
|
card_index_model_ids = parse_card_index_model_ids(card_index) if card_index is not None else set()
|
|
|
|
cases_by_model = {}
|
|
case_types_by_model = {}
|
|
for case in regression_cases:
|
|
model_id = case.get("model_id")
|
|
if not model_id:
|
|
continue
|
|
cases_by_model[model_id] = cases_by_model.get(model_id, 0) + 1
|
|
case_types_by_model.setdefault(model_id, set()).add(case.get("case_type"))
|
|
|
|
if model_index:
|
|
for entry in model_index["models"]:
|
|
entry_id = entry.get("model_id")
|
|
expected_case_count = cases_by_model.get(entry_id, 0)
|
|
actual_case_count = entry.get("regression_case_count")
|
|
if actual_case_count != expected_case_count:
|
|
errors.append(f"models/model_index.json entry {entry_id} regression_case_count is {actual_case_count}, expected {expected_case_count}")
|
|
if card_index is not None and entry_id not in card_index_model_ids:
|
|
errors.append(f"models/model_index.json entry {entry_id} missing from cards/card_index.md")
|
|
|
|
if model_index and card_index is not None:
|
|
for card_model_id in sorted(card_index_model_ids):
|
|
if card_model_id not in indexed_model_ids:
|
|
errors.append(f"cards/card_index.md entry {card_model_id} missing from models/model_index.json")
|
|
|
|
card_paths = sorted((root / "cards").glob("*.md")) if (root / "cards").exists() else []
|
|
for card_path in card_paths:
|
|
if card_path.name in {"README.md", "card_index.md"}:
|
|
continue
|
|
card_model_id = card_path.stem
|
|
if card_index is not None and card_model_id not in card_index_model_ids:
|
|
errors.append(f"card {card_path.relative_to(root).as_posix()} missing from cards/card_index.md")
|
|
|
|
for _, model in model_records:
|
|
model_id = model.get("model_id")
|
|
if not model_id:
|
|
continue
|
|
if model_index and model_id not in indexed_model_ids:
|
|
errors.append(f"model {model_id} missing from models/model_index.json")
|
|
if card_index is not None and model_id not in card_index_model_ids:
|
|
errors.append(f"model {model_id} missing from cards/card_index.md")
|
|
case_count = cases_by_model.get(model_id, 0)
|
|
if case_count < MIN_REGRESSION_CASES_PER_MODEL:
|
|
errors.append(f"model {model_id} has {case_count} regression cases, expected at least {MIN_REGRESSION_CASES_PER_MODEL}")
|
|
present_case_types = case_types_by_model.get(model_id, set())
|
|
for required_case_type in sorted(REQUIRED_REGRESSION_CASE_TYPES):
|
|
if required_case_type not in present_case_types:
|
|
errors.append(f"model {model_id} missing regression case type {required_case_type}")
|
|
|
|
return errors
|
|
|
|
|
|
def write_report(root, errors):
|
|
report_path = Path(root) / "reports" / "validation_report.md"
|
|
report_path.parent.mkdir(parents=True, exist_ok=True)
|
|
status = "PASS" if not errors else "FAIL"
|
|
lines = [
|
|
"# Validation Report",
|
|
"",
|
|
f"Status: `{status}`",
|
|
"",
|
|
"Command: `python scripts/validate_model_library.py`",
|
|
"",
|
|
]
|
|
if errors:
|
|
lines.append("## Errors")
|
|
lines.append("")
|
|
lines.extend(f"- {error}" for error in errors)
|
|
else:
|
|
lines.append("## Result")
|
|
lines.append("")
|
|
lines.append("- JSON 文件可解析。")
|
|
lines.append("- 模型必填字段存在并符合本地 contract 子集。")
|
|
lines.append("- source article、source excerpt、regression case 引用完整。")
|
|
lines.append("- model/card index 引用完整，计数、状态和索引投影一致。")
|
|
report_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
return report_path
|
|
|
|
|
|
def main():
|
|
root = Path(__file__).resolve().parents[1]
|
|
errors = validate_library(root)
|
|
report_path = write_report(root, errors)
|
|
print(f"validation report written to {report_path}")
|
|
if errors:
|
|
for error in errors:
|
|
print(f"ERROR: {error}")
|
|
return 1
|
|
print("validation passed")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|