diff --git a/README.md b/README.md index 6a70044..d0e9406 100644 --- a/README.md +++ b/README.md @@ -170,10 +170,12 @@ Indexes follow `docs/INDEX_MAINTENANCE_PROTOCOL.md`: every asset change must syn CCRA review packages are archived by round under `ccra_review_bundle/round-NN_YYYY-MM-DD_topic/`. Do not place new review bundle files directly in `ccra_review_bundle/`; create or update the current round directory instead. +Files intended for GPT / CCRA upload inside a review bundle should include the round version in the filename, for example `_03.2`. Raw changed-file zips should also include the round version while preserving source-relative paths inside the archive. + ## 13. Next Steps -1. Submit the Round 03.1 selector/no-call/regression patch for GPT / CCRA review. +1. Submit the Round 03.2 selector negative-gate patch for GPT / CCRA review. 2. Keep QPI and Intellectual Archaeology at `draft / B / pending` until Owner / CCRA review accepts stronger status. -3. Run a QPI blind-test round with new unlabelled inputs after Round 03.1 passes. +3. Run a QPI blind-test round with new unlabelled inputs only after Round 03.2 passes. 4. Defer any third-model expansion until the current QPI selector and case layer are accepted. 5. Route missing reusable extraction, inspection, or stability-scoring tools to `requirements/skills-vault/` or `requirements/ccpe/` instead of improvising local platform features. diff --git a/cards/intellectual_archaeology.md b/cards/intellectual_archaeology.md index 83caa71..991abd6 100644 --- a/cards/intellectual_archaeology.md +++ b/cards/intellectual_archaeology.md @@ -175,7 +175,7 @@ - 机制稳定性:中等,七层结构清晰,但层级之间的真实递进关系需要更多案例验证。 - 边界清晰度:中等,最小充分下潜原则已明确,但执行时容易过度下钻。 - 来源证据质量:较高,核心定义和七层结构来自原文,停止条件来自综合应用材料。 -- 回归测试表现:pending,已有五条样板用例,尚未扩展到更多领域。 +- 回归测试表现:pending,当前已有 19 条回归用例,已覆盖 positive、boundary、misuse、no_call、selector_gate 和 pipeline;Round 03.2 已纳入明确拒绝思想考古的模型级 hard exclusion 陷阱。 评级理由:纵向下潜机制清晰,适合深度建模,但七层结构和停止条件需要进一步稳定。 @@ -185,13 +185,10 @@ `pending` -当前已有五条回归用例: +当前已有 19 条回归用例,覆盖 positive、boundary、misuse、no_call、selector_gate 和 pipeline。Round 03.2 重点新增: -- `case_ia_positive_deep_modeling_001` -- `case_ia_positive_resilience_001` -- `case_ia_boundary_minimum_depth_001` -- `case_ia_misuse_philosophizing_001` -- `case_ia_misuse_no_validation_001` +- `case_ia_negative_explicit_do_not_call_001` +- `case_ia_negative_only_qpi_001` ## 示例输入 diff --git a/cards/qpi.md b/cards/qpi.md index 7648bc5..c659c1d 100644 --- a/cards/qpi.md +++ b/cards/qpi.md @@ -202,7 +202,7 @@ QPI 的三分结构、核心匮乏物、主体性和动态性有清晰来源支 - 机制稳定性:中高,扫描匮乏物和匹配处理范式的机制明确。 - 边界清晰度:中等,混合型问题、多视角分歧和上下文不足场景仍需更多测试。 - 来源证据质量:较高,主体性和动态性来自 2025 原文,核心匮乏物和误框定规则来自 2026 原文,应用规则来自综合文档。 -- 回归测试表现:pending,当前已有 52 条 QPI 回归用例;其中 Round 03 / 03.1 已纳入 owner-reviewed case layer、selector no-call、multi-perspective 和低上下文 provisional 边界用例。生命周期仍保持 draft。 +- 回归测试表现:pending,当前已有 55 条 QPI 回归用例;其中 Round 03 / 03.1 已纳入 owner-reviewed case layer、selector no-call、multi-perspective 和低上下文 provisional 边界用例,Round 03.2 已纳入普通 how-to / 词义判断 no-call 陷阱。生命周期仍保持 draft。 评级理由:三分结构清晰,适合作为入口路由模型,但需要补充大量边界案例,防止过度升维或降维。 @@ -212,7 +212,7 @@ QPI 的三分结构、核心匮乏物、主体性和动态性有清晰来源支 `pending` -当前已有 52 条 QPI 回归用例,覆盖 positive、boundary、misuse、no_call、selector_gate 和 pipeline。Round 03 / 03.1 重点新增: +当前已有 55 条 QPI 回归用例,覆盖 positive、boundary、misuse、no_call、selector_gate 和 pipeline。Round 03 / 03.1 重点新增: - owner-reviewed flow / disappointment / organizational year-end-review cases; - `case_qpi_international_logistics_no_call_001` @@ -222,6 +222,12 @@ QPI 的三分结构、核心匮乏物、主体性和动态性有清晰来源支 - `case_qpi_direct_summary_no_call_001` - `case_qpi_analysis_override_should_call_001` +Round 03.2 重点新增: + +- `case_qpi_how_to_excel_no_call_001` +- `case_qpi_how_to_markdown_pdf_no_call_001` +- `case_qpi_dictionary_judgment_no_call_001` + ## 示例输入 - “我们团队每次都说要长期主义,但一到季度 KPI 就回到短期动作,这到底是什么问题?” @@ -280,7 +286,7 @@ QPI 的三分结构、核心匮乏物、主体性和动态性有清晰来源支 QPI 应作为问题回答系统的前置路由模型,用于防止系统在问题类型错误的情况下直接给答案。 -selector 应优先使用 `trigger_keywords`、`negative_triggers`、`pipeline_position` 和 `selection_priority` 做规则推荐。 +selector 应优先使用 `trigger_keywords`、`negative_triggers`、模型级 hard exclusion、`pipeline_position` 和 `selection_priority` 做规则推荐。 ## 版本信息 diff --git a/ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/00_OPEN_THIS_FIRST_CCRA_REVIEW_BRIEF.md b/ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/00_OPEN_THIS_FIRST_CCRA_REVIEW_BRIEF03.1.md similarity index 100% rename from ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/00_OPEN_THIS_FIRST_CCRA_REVIEW_BRIEF.md rename to ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/00_OPEN_THIS_FIRST_CCRA_REVIEW_BRIEF03.1.md diff --git a/ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/01_PATCH_MATRIX.md b/ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/01_PATCH_MATRIX_03.1.md similarity index 100% rename from ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/01_PATCH_MATRIX.md rename to ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/01_PATCH_MATRIX_03.1.md diff --git a/ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/02_CURRENT_ASSET_PACK.md b/ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/02_CURRENT_ASSET_PACK03.1.md similarity index 100% rename from ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/02_CURRENT_ASSET_PACK.md rename to ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/02_CURRENT_ASSET_PACK03.1.md diff --git a/ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/03_VALIDATION_AND_COMMAND_LOG.md b/ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/03_VALIDATION_AND_COMMAND_LOG03.1.md similarity index 100% rename from ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/03_VALIDATION_AND_COMMAND_LOG.md rename to ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/03_VALIDATION_AND_COMMAND_LOG03.1.md diff --git a/ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/04_REVIEW_QUESTIONS_FOR_GPT.md b/ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/04_REVIEW_QUESTIONS_FOR_GPT03.1.md similarity index 100% rename from ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/04_REVIEW_QUESTIONS_FOR_GPT.md rename to ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/04_REVIEW_QUESTIONS_FOR_GPT03.1.md diff --git a/ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/BUNDLE_FILE_MANIFEST.md b/ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/BUNDLE_FILE_MANIFEST03.1.md similarity index 100% rename from ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/BUNDLE_FILE_MANIFEST.md rename to ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/BUNDLE_FILE_MANIFEST03.1.md diff --git a/ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/optional_raw_changed_files.zip b/ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/optional_raw_changed_files03.1.zip similarity index 100% rename from ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/optional_raw_changed_files.zip rename to ccra_review_bundle/round-03.1_2026-06-17_selector-no-call-regression-patch/optional_raw_changed_files03.1.zip diff --git a/ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/00_OPEN_THIS_FIRST_CCRA_REVIEW_BRIEF_03.2.md b/ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/00_OPEN_THIS_FIRST_CCRA_REVIEW_BRIEF_03.2.md new file mode 100644 index 0000000..35e2f32 --- /dev/null +++ b/ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/00_OPEN_THIS_FIRST_CCRA_REVIEW_BRIEF_03.2.md @@ -0,0 +1,55 @@ +# Round 03.2 CCRA Review Brief + +Date: 2026-06-17 + +Repository: `C:\Users\wangq\Documents\Codex\work-projects\the-mindscape-of-bro-tsong` + +Patch title: Selector Negative-Gate Patch + +## Scope + +Round 03.2 is a tiny follow-up to Round 03.1. + +It addresses GPT review finding: + +> IA / 思想考古 negative triggers were still score penalties, not model-level hard exclusions. + +This patch does not: + +- add a third model; +- upgrade QPI or Intellectual Archaeology lifecycle status; +- introduce an LLM selector; +- introduce RAG, database, frontend, backend, user accounts, or platform features; +- start Round 04 blind testing. + +Current model status remains: + +```text +qpi: draft / B / pending +intellectual_archaeology: draft / B / pending +``` + +## What Changed + +1. Added model-level hard exclusion for explicit IA refusal. +2. Removed bare QPI gate signals `如何` and `判断`. +3. Replaced broad gates with compound problem-definition signals. +4. Strengthened selector calibration smoke checks: + - `no_call` requires `selected_models == []` and `no_call == true`; + - `no_call_or_low_priority` rejects both QPI and IA; + - `select_qpi_reject_ia` requires QPI selected and IA rejected; + - IA selection expectations fail if IA hard exclusion is present. +5. Added five GPT-requested traps to regression and calibration data. + +## Suggested Review Order + +1. `00_OPEN_THIS_FIRST_CCRA_REVIEW_BRIEF_03.2.md` +2. `01_PATCH_MATRIX_03.2.md` +3. `02_CURRENT_ASSET_PACK_03.2.md` +4. `03_VALIDATION_AND_COMMAND_LOG_03.2.md` +5. `04_REVIEW_QUESTIONS_FOR_GPT_03.2.md` +6. `optional_raw_changed_files_03.2.zip` only if exact changed-file inspection is needed + +## Primary Review Question + +Does Round 03.2 close the model-level negative gate issue enough to close Round 03 and proceed to Round 04 blind input evaluation? diff --git a/ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/01_PATCH_MATRIX_03.2.md b/ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/01_PATCH_MATRIX_03.2.md new file mode 100644 index 0000000..aecac1e --- /dev/null +++ b/ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/01_PATCH_MATRIX_03.2.md @@ -0,0 +1,53 @@ +# Round 03.2 Patch Matrix + +## GPT Requirement Mapping + +| Requirement | Implementation | Files | +| --- | --- | --- | +| IA explicit refusal must be hard exclusion, not score penalty | Added `hard_exclusion_triggers` for IA and forced rejected decision when matched | `selector/selector_rules.json`, `scripts/run_selector_demo.py` | +| Include refusal phrases such as `不要思想考古`, `先不要思想考古`, `不调用思想考古`, `只做 QPI`, `只做问题定性` | Added IA hard exclusion trigger list | `selector/selector_rules.json` | +| Preserve QPI explicit-analysis override but do not let it re-enable IA | Hard exclusion applies per model after scoring and is not bypassed by QPI override | `scripts/run_selector_demo.py` | +| Remove or narrow bare `如何` and `判断` QPI gates | Removed bare signals; added compound problem-definition phrases | `selector/selector_rules.json` | +| Strict `no_call` smoke behavior | `no_call` now requires no selected models and `no_call=true` | `scripts/run_selector_calibration_smoke.py` | +| Strict `no_call_or_low_priority` behavior | Fails if QPI or IA is selected | `scripts/run_selector_calibration_smoke.py` | +| Add `select_qpi_reject_ia` behavior | Requires QPI selected and IA rejected | `scripts/run_selector_calibration_smoke.py`, `selector/selector_calibration_inputs.json` | +| Fail IA selection expectations if IA hard exclusion exists | Added hard-exclusion check for `select_intellectual_archaeology` expected cases | `scripts/run_selector_calibration_smoke.py` | +| Add five traps | Added two IA hard-exclusion traps and three generic how-to / dictionary no-call traps | `tests/*.regression.json`, `selector/selector_calibration_inputs.json` | + +## Added Regression Cases + +### Intellectual Archaeology + +- `case_ia_negative_explicit_do_not_call_001` +- `case_ia_negative_only_qpi_001` + +### QPI + +- `case_qpi_how_to_excel_no_call_001` +- `case_qpi_how_to_markdown_pdf_no_call_001` +- `case_qpi_dictionary_judgment_no_call_001` + +## Added Calibration Inputs + +- `selector_calibration_ia_negative_explicit_do_not_call_001` +- `selector_calibration_ia_negative_only_qpi_001` +- `selector_calibration_qpi_how_to_excel_no_call_001` +- `selector_calibration_qpi_how_to_markdown_pdf_no_call_001` +- `selector_calibration_qpi_dictionary_judgment_no_call_001` + +## Current Counts + +| Asset | Round 03.1 | Round 03.2 | +| --- | ---: | ---: | +| QPI regression cases | 52 | 55 | +| IA regression cases | 17 | 19 | +| Aggregate regression cases | 69 | 74 | +| Selector calibration inputs | 85 | 90 | + +## Non-Goals Preserved + +- No new model. +- No lifecycle upgrade. +- No LLM selector. +- No RAG / database / frontend / backend. +- No Round 04 blind-test execution. diff --git a/ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/02_CURRENT_ASSET_PACK_03.2.md b/ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/02_CURRENT_ASSET_PACK_03.2.md new file mode 100644 index 0000000..28bc704 --- /dev/null +++ b/ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/02_CURRENT_ASSET_PACK_03.2.md @@ -0,0 +1,73 @@ +# Round 03.2 Current Asset Pack + +## Canonical Changed Assets + +- `selector/selector_rules.json` +- `selector/selector_calibration_inputs.json` +- `tests/qpi.regression.json` +- `tests/intellectual_archaeology.regression.json` +- `tests/regression_cases.json` +- `tests/regression_index.json` +- `scripts/run_selector_demo.py` +- `scripts/run_selector_calibration_smoke.py` + +## Generated / Derived Assets + +- `models/model_index.json` +- `cards/card_index.md` +- `reports/index_rebuild_report.md` +- `reports/validation_report.md` +- `reports/selector_regression_report_v0.2.md` +- `reports/selector_calibration_smoke_report.md` +- `reports/model_card_sync_report_v0.2.md` + +## Documentation Updated + +- `README.md` +- `docs/DECISIONS.md` +- `selector/README.md` +- `scripts/README.md` +- `reports/README.md` +- `cards/qpi.md` +- `cards/intellectual_archaeology.md` +- `reports/content_review_report_v0.2.md` + +## Selector Behavior Summary + +### IA explicit refusal + +Inputs such as: + +```text +QPI 已判断:这是 Issue。先不要思想考古,只做问题定性复核。 +不要思想考古,只做 QPI 复核。 +``` + +now select QPI and reject `intellectual_archaeology`. + +### Generic how-to / dictionary no-call + +Inputs such as: + +```text +如何在 Excel 里冻结首行? +如何把 Markdown 转 PDF? +判断这个词是什么意思。 +``` + +now produce no selected models and `no_call=true`. + +### Still preserved + +QPI remains callable for explicit problem-definition or owner-reviewed calibration inputs. + +IA remains callable when IA is explicitly requested without hard exclusion and the heavy-depth / QPI-completed gate is satisfied. + +## Current Lifecycle State + +No lifecycle upgrade was made. + +```text +qpi: draft / B / pending +intellectual_archaeology: draft / B / pending +``` diff --git a/ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/03_VALIDATION_AND_COMMAND_LOG_03.2.md b/ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/03_VALIDATION_AND_COMMAND_LOG_03.2.md new file mode 100644 index 0000000..29625a2 --- /dev/null +++ b/ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/03_VALIDATION_AND_COMMAND_LOG_03.2.md @@ -0,0 +1,61 @@ +# Round 03.2 Validation and Command Log + +Date: 2026-06-17 + +## Red Evidence + +After adding the five GPT-requested traps and before implementation, selector checks failed as expected. + +`python scripts\run_selector_regression.py` + +- `case_ia_negative_explicit_do_not_call_001`: IA was selected despite explicit refusal. +- `case_ia_negative_only_qpi_001`: QPI was not selected. +- `case_qpi_how_to_excel_no_call_001`: QPI was selected. +- `case_qpi_how_to_markdown_pdf_no_call_001`: QPI was selected. +- `case_qpi_dictionary_judgment_no_call_001`: QPI was selected. + +`python scripts\run_selector_calibration_smoke.py` + +- `selector_calibration_ia_negative_explicit_do_not_call_001`: expected IA rejection. +- `selector_calibration_ia_negative_only_qpi_001`: expected QPI selection. +- three generic how-to / dictionary traps expected no QPI selection. + +## Green Verification + +The following commands passed after the Round 03.2 patch: + +```powershell +python scripts\rebuild_indexes.py --check +python -m unittest discover -s tests -p "test*.py" -v +python scripts\validate_model_library.py +python scripts\check_card_contract.py +python scripts\run_selector_demo.py +python scripts\run_selector_regression.py +python scripts\run_selector_calibration_smoke.py +python scripts\check_model_card_sync.py +``` + +Results: + +- Index check: PASS +- Unit tests: PASS, 17 tests +- Model library validation: PASS +- Card contract: PASS +- Selector demo: PASS +- Selector regression: PASS, 18 selector-relevant cases +- Selector calibration smoke: PASS, 90 inputs, 0 failures, 0 warnings +- Model/card sync: PASS + +## Generated Reports + +- `reports/index_rebuild_report.md` +- `reports/validation_report.md` +- `reports/selector_regression_report_v0.2.md` +- `reports/selector_calibration_smoke_report.md` +- `reports/model_card_sync_report_v0.2.md` + +## Important Interpretation + +Validation pass does not mean model content is stable or owner-approved. + +It only means the Round 03.2 selector negative-gate patch is structurally consistent and passes current local checks. diff --git a/ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/04_REVIEW_QUESTIONS_FOR_GPT_03.2.md b/ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/04_REVIEW_QUESTIONS_FOR_GPT_03.2.md new file mode 100644 index 0000000..638a9b3 --- /dev/null +++ b/ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/04_REVIEW_QUESTIONS_FOR_GPT_03.2.md @@ -0,0 +1,29 @@ +# Round 03.2 Review Questions for GPT + +Please review Round 03.2 only. + +Do not expand into Round 04 blind testing yet. + +## Questions + +1. Does model-level hard exclusion now correctly handle explicit IA refusal? +2. Are the current hard exclusion phrases sufficient for Round 03 closure? +3. Is it acceptable that hard exclusion rejects only IA while still allowing QPI when the user says `只做 QPI` or `只做问题定性`? +4. Is removing bare `如何` and `判断` while preserving compound problem-definition phrases enough to reduce QPI over-selection? +5. Are the strengthened calibration smoke assertions strict enough to become a long-term selector gate? +6. Can Round 03 now be closed with the following conclusion? + +```text +QPI and Intellectual Archaeology remain draft / B / pending. +QPI is accepted as a draft-callable routing asset at report level. +Selector v0.2 passes no-call, calibration smoke, model-level exclusion, and QPI-before-IA gates. +Proceed to Round 04 blind input evaluation. +``` + +## Suggested Answer Format + +Please answer with: + +- `pass`, `revise`, or `block`; +- specific failing cases if `revise` or `block`; +- whether Round 04 blind input evaluation may start after this patch. diff --git a/ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/BUNDLE_FILE_MANIFEST_03.2.md b/ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/BUNDLE_FILE_MANIFEST_03.2.md new file mode 100644 index 0000000..1979589 --- /dev/null +++ b/ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/BUNDLE_FILE_MANIFEST_03.2.md @@ -0,0 +1,52 @@ +# Round 03.2 Bundle File Manifest + +## Review Files + +- `00_OPEN_THIS_FIRST_CCRA_REVIEW_BRIEF_03.2.md` +- `01_PATCH_MATRIX_03.2.md` +- `02_CURRENT_ASSET_PACK_03.2.md` +- `03_VALIDATION_AND_COMMAND_LOG_03.2.md` +- `04_REVIEW_QUESTIONS_FOR_GPT_03.2.md` +- `BUNDLE_FILE_MANIFEST_03.2.md` +- `optional_raw_changed_files_03.2.zip` + +## Optional Raw Changed Files + +The zip is optional and should be used only if exact file inspection is needed. + +It preserves source-relative paths. + +It excludes: + +- `knowledge_assets/` +- `__pycache__/` +- `*.pyc` +- previous review bundle zip files + +## Source-Relative Files Included in Raw Zip + +- `README.md` +- `cards/intellectual_archaeology.md` +- `cards/qpi.md` +- `docs/DECISIONS.md` +- `docs/FILE_TAXONOMY.md` +- `reports/README.md` +- `reports/content_review_report_v0.2.md` +- `reports/Codex新会话交接文档_Round03.2评审准备_2026-06-17.md` +- `reports/index_rebuild_report.md` +- `reports/model_card_sync_report_v0.2.md` +- `reports/selector_calibration_smoke_report.md` +- `reports/selector_regression_report_v0.2.md` +- `reports/validation_report.md` +- `scripts/README.md` +- `scripts/run_selector_calibration_smoke.py` +- `scripts/run_selector_demo.py` +- `selector/README.md` +- `selector/selector_calibration_inputs.json` +- `selector/selector_rules.json` +- `tests/intellectual_archaeology.regression.json` +- `tests/qpi.regression.json` +- `tests/regression_cases.json` +- `tests/regression_index.json` +- `models/model_index.json` +- `cards/card_index.md` diff --git a/ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/optional_raw_changed_files_03.2.zip b/ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/optional_raw_changed_files_03.2.zip new file mode 100644 index 0000000..31e040f Binary files /dev/null and b/ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/optional_raw_changed_files_03.2.zip differ diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index 72552b9..0280494 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -156,5 +156,28 @@ This keeps second-round and third-round materials comparable without overwriting Round 03.1 added two bundle hygiene rules: -- `optional_raw_changed_files.zip` must preserve source-relative paths. Do not create flat archives that can collide on names such as `README.md`. +- `optional_raw_changed_files_.zip` must preserve source-relative paths. Do not create flat archives that can collide on names such as `README.md`. - `knowledge_assets/` is excluded from review bundles by default because the project owner manually syncs stable long-term knowledge assets into GPT knowledge storage. Include it only when the current review explicitly targets the knowledge asset itself. + +Round 03.2 added one ChatGPT upload hygiene rule: + +- Every file intended for GPT / CCRA upload inside a review bundle must include the round version in the filename, such as `_03.2` or `03.2`, because uploading same-named files from multiple rounds in one Web ChatGPT conversation can cause file-reading ambiguity. This applies especially to raw changed-file zip archives, e.g. `optional_raw_changed_files_03.2.zip`. +- The version suffix belongs to the bundle file names. Zip entries should still preserve source-relative repository paths so reviewers can inspect actual changed files without flattening path context. + +## Decision 014: Treat explicit model refusal as model-level hard exclusion + +Status: Accepted + +Reason: + +Round 03.1 fixed QPI default over-selection, but GPT review found that explicit Intellectual Archaeology refusal phrases such as `先不要思想考古` could still lose to positive IA signals and QPI-completed bonuses. + +A user instruction that names a model and refuses it should not be represented as a normal score penalty. It must hard-exclude that model unless a later owner-approved rule defines an explicit reversal phrase. + +The current Round 03.2 implementation keeps this scoped: + +- It rejects only the explicitly refused model. +- It does not create a global no-call. +- It does not add an LLM selector, RAG, database, service, frontend, backend, or third model. +- It preserves QPI explicit-analysis override, but QPI selection does not re-enable IA when IA is explicitly refused. +- Generic bare QPI gate words such as `如何` and `判断` are removed in favor of compound problem-definition phrases. diff --git a/docs/FILE_TAXONOMY.md b/docs/FILE_TAXONOMY.md index 200cab0..aac69cd 100644 --- a/docs/FILE_TAXONOMY.md +++ b/docs/FILE_TAXONOMY.md @@ -97,7 +97,8 @@ Rules: - Keep review bundles under dated round directories. - Do not place new review files directly under `ccra_review_bundle/`. -- `optional_raw_changed_files.zip` must preserve source-relative paths. +- `optional_raw_changed_files_.zip` must preserve source-relative paths. +- GPT / CCRA upload files inside review bundles must include the round version in the filename, for example `_03.2`, to avoid same-name ambiguity when multiple rounds are uploaded in one Web ChatGPT conversation. - Do not flatten zip contents, because duplicate filenames such as `README.md` can collide. - `knowledge_assets/` is excluded from review zips by default because the Owner manually syncs stable knowledge assets into GPT knowledge storage. diff --git a/models/model_index.json b/models/model_index.json index 9a41e2e..dee950e 100644 --- a/models/model_index.json +++ b/models/model_index.json @@ -11,7 +11,7 @@ "card_file": "cards/intellectual_archaeology.md", "source_article_count": 2, "source_evidence_count": 5, - "regression_case_count": 17, + "regression_case_count": 19, "stability_level": "B", "regression_status": "pending", "status": "draft", @@ -26,7 +26,7 @@ "card_file": "cards/qpi.md", "source_article_count": 3, "source_evidence_count": 10, - "regression_case_count": 52, + "regression_case_count": 55, "stability_level": "B", "regression_status": "pending", "status": "draft", diff --git a/reports/Codex新会话交接文档_Round03.2评审准备_2026-06-17.md b/reports/Codex新会话交接文档_Round03.2评审准备_2026-06-17.md new file mode 100644 index 0000000..ee6cc6a --- /dev/null +++ b/reports/Codex新会话交接文档_Round03.2评审准备_2026-06-17.md @@ -0,0 +1,125 @@ +# Codex 新会话交接文档:Round 03.2 评审准备 + +Date: 2026-06-17 + +Repository: `C:\Users\wangq\Documents\Codex\work-projects\the-mindscape-of-bro-tsong` + +Current phase: `model_library_mvp` + +## 1. 本轮目标 + +根据 GPT 对 Round 03.1 的 `REVISE` 评审意见,完成一个极小的 Round 03.2 selector negative-gate patch,并准备 03.2 评审资料。 + +## 2. 已完成 + +- 将 IA / 思想考古的明确拒绝语从普通 penalty 升级为 model-level hard exclusion。 +- 保留 QPI 显式分析 / 复核入口,但不允许 QPI 选择重新启用被明确拒绝的 IA。 +- 移除 QPI gate 中裸 `如何` 和裸 `判断`。 +- 增加复合 problem-definition 信号,避免普通 how-to / 词义判断误召回 QPI。 +- 加严 `scripts/run_selector_calibration_smoke.py`: + - `no_call` 必须 `selected_models == []` 且 `no_call == true`; + - `no_call_or_low_priority` 不允许 QPI 或 IA 被选中; + - `select_qpi_reject_ia` 必须选中 QPI 且拒绝 IA; + - `select_intellectual_archaeology` 如果命中 IA hard exclusion 则失败。 +- 增加 5 条 GPT 指定陷阱: + - `case_ia_negative_explicit_do_not_call_001` + - `case_ia_negative_only_qpi_001` + - `case_qpi_how_to_excel_no_call_001` + - `case_qpi_how_to_markdown_pdf_no_call_001` + - `case_qpi_dictionary_judgment_no_call_001` +- 同步 per-model regression、aggregate regression、regression index、model/card index 和相关报告。 +- 生成 Round 03.2 评审包: + +```text +ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/ +``` +- Round 03.2 评审包内上传给 GPT / CCRA 的文件名均带 `03.2` 版本号;raw zip 命名为 `optional_raw_changed_files_03.2.zip`,zip 内保持 source-relative paths。 + +## 3. 当前计数 + +- QPI regression cases: 55 +- Intellectual Archaeology regression cases: 19 +- Aggregate regression cases: 74 +- Selector calibration inputs: 90 + +模型状态没有升级: + +```text +qpi: draft / B / pending +intellectual_archaeology: draft / B / pending +``` + +## 4. 主要变更文件 + +- `selector/selector_rules.json` +- `scripts/run_selector_demo.py` +- `scripts/run_selector_calibration_smoke.py` +- `selector/selector_calibration_inputs.json` +- `tests/qpi.regression.json` +- `tests/intellectual_archaeology.regression.json` +- `tests/regression_cases.json` +- `tests/regression_index.json` +- `models/model_index.json` +- `cards/card_index.md` +- `cards/qpi.md` +- `cards/intellectual_archaeology.md` +- `docs/DECISIONS.md` +- `README.md` +- `selector/README.md` +- `scripts/README.md` +- `reports/README.md` +- `reports/content_review_report_v0.2.md` +- generated reports under `reports/` +- Round 03.2 review bundle under `ccra_review_bundle/round-03.2_2026-06-17_selector-negative-gate-patch/` + +## 5. 验证结果 + +以下命令已通过: + +```powershell +python scripts\rebuild_indexes.py --check +python -m unittest discover -s tests -p "test*.py" -v +python scripts\validate_model_library.py +python scripts\check_card_contract.py +python scripts\run_selector_demo.py +python scripts\run_selector_regression.py +python scripts\run_selector_calibration_smoke.py +python scripts\check_model_card_sync.py +``` + +结果: + +- Index check: PASS +- Unit tests: PASS, 17 tests +- Model library validation: PASS +- Card contract: PASS +- Selector demo: PASS +- Selector regression: PASS +- Selector calibration smoke: PASS, 90 inputs, 0 failures, 0 warnings +- Model/card sync: PASS + +## 6. 假设 + +- GPT 的 `Round 03.2 Selector Negative-Gate Patch` 指令是本轮唯一修复范围。 +- `只做 QPI` 和 `只做问题定性` 在当前上下文中表示拒绝 IA,同时允许 QPI。 +- `draft-callable` 只能作为评审报告语言,不写入模型 JSON status。 + +## 7. 仍不代表已经完成的事 + +- 没有进入 Round 04 blind test。 +- 没有证明 QPI 或 IA 内容已稳定。 +- 没有升级 `status`、`stability_level` 或 `regression_status`。 +- 没有新增第三模型、LLM selector、RAG、数据库、前后端或完整问答系统。 + +## 8. 需要 GPT / Owner 判断 + +- Round 03.2 是否 `pass / revise / block`。 +- Round 03 是否可以正式关闭。 +- 是否允许进入 Round 04 blind input evaluation。 +- 当前 IA hard exclusion 词表是否足够,是否需要定义“用户反转前文拒绝”的显式规则。 + +## 9. 建议下一步 + +1. 将 Round 03.2 review bundle 提交给 GPT。 +2. 如果 GPT pass,关闭 Round 03 系列。 +3. 仅在 GPT / Owner 同意后启动 Round 04 blind input evaluation。 diff --git a/reports/README.md b/reports/README.md index 3c085b0..915d5a9 100644 --- a/reports/README.md +++ b/reports/README.md @@ -10,6 +10,7 @@ Current reports: - `baseline_before_content_stabilization_v0.2.md` - `evidence_coverage_report_v0.2.md` - `selector_regression_report_v0.2.md` +- `selector_calibration_smoke_report.md` - `model_card_sync_report_v0.2.md` - `content_review_report_v0.2.md` - `model_review_status.json` diff --git a/reports/content_review_report_v0.2.md b/reports/content_review_report_v0.2.md index c9801cc..52834c0 100644 --- a/reports/content_review_report_v0.2.md +++ b/reports/content_review_report_v0.2.md @@ -2,7 +2,7 @@ Date: 2026-06-16 -Status note: this report is a pre-case-promotion review snapshot. It remains useful for the v0.2 evidence / contract / selector baseline, but its regression counts do not reflect the Round 03 / 03.1 QPI owner-reviewed case promotion. Current QPI regression counts are tracked in `tests/qpi.regression.json`, `models/model_index.json`, and the Round 03.1 review bundle. +Status note: this report is a pre-case-promotion review snapshot. It remains useful for the v0.2 evidence / contract / selector baseline, but its regression counts do not reflect the Round 03 / 03.1 QPI owner-reviewed case promotion or the Round 03.2 selector negative-gate patch. Current regression counts are tracked in `tests/*.regression.json`, `models/model_index.json`, and the latest review bundle. ## 1. 本轮修改摘要 @@ -111,7 +111,7 @@ Coverage: - 已覆盖误用、防误召回、no-call 和 pipeline gate。 - 仍需人工审查用例真实性和遗漏边界。 -Round 03 / 03.1 update: QPI has since expanded to 52 regression cases after owner-reviewed case promotion and selector/no-call repair. This v0.2 table is retained as a historical baseline, not the current count. +Round 03 / 03.1 / 03.2 update: QPI has since expanded to 55 regression cases after owner-reviewed case promotion, selector/no-call repair, and broad-gate no-call traps. Intellectual Archaeology has expanded to 19 regression cases after Round 03.2 IA hard-exclusion traps. This v0.2 table is retained as a historical baseline, not the current count. ## 6. Selector Regression 结论 @@ -124,10 +124,13 @@ Selector v0.2 仍为规则 selector。 - QPI-before-IA gate - score breakdown - rejected model reasons +- model-level hard exclusion for explicit model refusal +- stricter calibration smoke checks for no-call, no-call-or-low-priority, and select-qpi-reject-ia 报告位于: - `reports/selector_regression_report_v0.2.md` +- `reports/selector_calibration_smoke_report.md` 结论: diff --git a/reports/selector_calibration_smoke_report.md b/reports/selector_calibration_smoke_report.md index 98cc8f4..6a4c630 100644 --- a/reports/selector_calibration_smoke_report.md +++ b/reports/selector_calibration_smoke_report.md @@ -4,8 +4,9 @@ Status: `PASS` Command: `python scripts/run_selector_calibration_smoke.py` -Calibration inputs checked: 85 +Calibration inputs checked: 90 Failures: 0 +Warnings: 0 ## Cases @@ -93,4 +94,9 @@ Failures: 0 - `selector_calibration_qpi_research_safety_accountability_001`: PASS; expected=select_qpi; selected=['qpi']; no_call=False - `selector_calibration_qpi_research_penalty_integrity_001`: PASS; expected=select_qpi; selected=['qpi']; no_call=False - `selector_calibration_direct_summary_no_call_001`: PASS; expected=no_call; selected=[]; no_call=True +- `selector_calibration_ia_negative_explicit_do_not_call_001`: PASS; expected=select_qpi_reject_ia; selected=['qpi']; no_call=False +- `selector_calibration_ia_negative_only_qpi_001`: PASS; expected=select_qpi_reject_ia; selected=['qpi']; no_call=False +- `selector_calibration_qpi_how_to_excel_no_call_001`: PASS; expected=no_call; selected=[]; no_call=True +- `selector_calibration_qpi_how_to_markdown_pdf_no_call_001`: PASS; expected=no_call; selected=[]; no_call=True +- `selector_calibration_qpi_dictionary_judgment_no_call_001`: PASS; expected=no_call; selected=[]; no_call=True - `selector_calibration_analysis_override_should_call_001`: PASS; expected=select_qpi; selected=['qpi']; no_call=False diff --git a/reports/selector_regression_report_v0.2.md b/reports/selector_regression_report_v0.2.md index dbce455..9ba39dc 100644 --- a/reports/selector_regression_report_v0.2.md +++ b/reports/selector_regression_report_v0.2.md @@ -2,7 +2,7 @@ Status: `PASS` -Cases checked: 13 +Cases checked: 18 Failures: 0 ## Cases @@ -13,10 +13,15 @@ Failures: 0 - `case_ia_selector_gate_qpi_first_001`: PASS; selected=['qpi']; no_call=False - `case_ia_selector_gate_no_heavy_signal_001`: PASS; selected=[]; no_call=True - `case_ia_pipeline_after_qpi_001`: PASS; selected=['intellectual_archaeology', 'qpi']; no_call=False +- `case_ia_negative_explicit_do_not_call_001`: PASS; selected=['qpi']; no_call=False +- `case_ia_negative_only_qpi_001`: PASS; selected=['qpi']; no_call=False - `case_qpi_no_call_direct_edit_001`: PASS; selected=[]; no_call=True - `case_qpi_selector_gate_fact_001`: PASS; selected=[]; no_call=True - `case_qpi_pipeline_before_ia_001`: PASS; selected=['qpi']; no_call=False - `case_qpi_disappointment_mismatch_diagnostics_001`: PASS; selected=['qpi']; no_call=False - `case_qpi_international_logistics_no_call_001`: PASS; selected=[]; no_call=True - `case_qpi_direct_summary_no_call_001`: PASS; selected=[]; no_call=True +- `case_qpi_how_to_excel_no_call_001`: PASS; selected=[]; no_call=True +- `case_qpi_how_to_markdown_pdf_no_call_001`: PASS; selected=[]; no_call=True +- `case_qpi_dictionary_judgment_no_call_001`: PASS; selected=[]; no_call=True - `case_qpi_analysis_override_should_call_001`: PASS; selected=['qpi']; no_call=False diff --git a/scripts/README.md b/scripts/README.md index 05b3b6c..b4e9cf4 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -10,6 +10,7 @@ Current script: - `check_model_card_sync.py` - `run_selector_regression.py` - `run_selector_demo.py` +- `run_selector_calibration_smoke.py` Run: @@ -44,7 +45,10 @@ Run selector regression and model/card sync checks: ```powershell python scripts\run_selector_regression.py +python scripts\run_selector_calibration_smoke.py python scripts\check_model_card_sync.py ``` +`run_selector_calibration_smoke.py` checks selector calibration inputs, including strict `no_call`, `no_call_or_low_priority`, `select_qpi_reject_ia`, and IA heavy-depth gate expectations. + Prefer Python standard library before adding dependencies. diff --git a/scripts/run_selector_calibration_smoke.py b/scripts/run_selector_calibration_smoke.py index 4e05681..36fe346 100644 --- a/scripts/run_selector_calibration_smoke.py +++ b/scripts/run_selector_calibration_smoke.py @@ -2,7 +2,7 @@ import json import sys from pathlib import Path -from run_selector_demo import load_selector_rules, recommend +from run_selector_demo import load_selector_rules, model_hard_exclusion_hits, recommend def read_json(path): @@ -24,9 +24,24 @@ def evaluate_input(root, selector_rules, item): selected_ids = selected_model_ids(result) expected = item.get("expected_selector_behavior") errors = [] + warnings = [] - if expected in {"no_call", "no_call_or_low_priority"} and "qpi" in selected_ids: - errors.append("expected no QPI selection") + model_rules = { + model_rule["model_id"]: model_rule + for model_rule in selector_rules.get("models", []) + } + + if expected == "no_call": + if selected_ids: + errors.append(f"expected no selected models, got {selected_ids}") + if result.get("no_call") is not True: + errors.append("expected no_call=true") + + if expected == "no_call_or_low_priority": + if "qpi" in selected_ids: + errors.append("expected no QPI selection") + if "intellectual_archaeology" in selected_ids: + errors.append("expected no IA selection") if expected in {"select_qpi", "select_qpi_low_confidence", "select_qpi_reject_ia"}: if "qpi" not in selected_ids: @@ -40,6 +55,15 @@ def evaluate_input(root, selector_rules, item): errors.append("expected IA selection") if not has_ia_gate(item["input"], selector_rules): errors.append("IA selected expectation lacks heavy-depth or QPI-completed gate") + ia_hard_exclusion_hits = model_hard_exclusion_hits( + item["input"], + model_rules.get("intellectual_archaeology", {}), + ) + if ia_hard_exclusion_hits: + errors.append("IA selection expected despite IA hard exclusion signal") + + if expected == "select_qpi" and set(selected_ids) - {"qpi"}: + warnings.append("unexpected extra selected models: " + ", ".join(sorted(set(selected_ids) - {"qpi"}))) return { "case_id": item["case_id"], @@ -47,12 +71,14 @@ def evaluate_input(root, selector_rules, item): "selected_models": selected_ids, "no_call": result.get("no_call"), "errors": errors, + "warnings": warnings, } def write_report(root, results): report_path = root / "reports" / "selector_calibration_smoke_report.md" failures = [result for result in results if result["errors"]] + warnings = [result for result in results if result["warnings"]] lines = [ "# Selector Calibration Smoke Report", "", @@ -62,6 +88,7 @@ def write_report(root, results): "", f"Calibration inputs checked: {len(results)}", f"Failures: {len(failures)}", + f"Warnings: {len(warnings)}", "", "## Cases", "", @@ -74,6 +101,8 @@ def write_report(root, results): ) for error in result["errors"]: lines.append(f" - {error}") + for warning in result["warnings"]: + lines.append(f" - WARNING: {warning}") report_path.write_text("\n".join(lines) + "\n", encoding="utf-8") return report_path diff --git a/scripts/run_selector_demo.py b/scripts/run_selector_demo.py index a8a295d..d928003 100644 --- a/scripts/run_selector_demo.py +++ b/scripts/run_selector_demo.py @@ -33,6 +33,10 @@ def hard_no_call_hits(user_input, global_rules): return hit_any(user_input, signals) +def model_hard_exclusion_hits(user_input, model_rule): + return hit_any(user_input, model_rule.get("hard_exclusion_triggers", [])) + + def score_model(model, model_rule, global_rules, user_input, task_type="", pipeline_position=""): model_id = model.get("model_id") weights = global_rules.get("weights", {}) @@ -146,10 +150,17 @@ def recommend(root, user_input, task_type="", pipeline_position="", threshold=No return hard_no_call_result(root, user_input, no_call_hits) model_rules = rules_by_model(selector_rules) - scored = [ - score_model(model, model_rules.get(model.get("model_id"), {}), global_rules, user_input, task_type, pipeline_position) - for model in load_models(root) - ] + scored = [] + for model in load_models(root): + model_id = model.get("model_id") + model_rule = model_rules.get(model_id, {}) + item = score_model(model, model_rule, global_rules, user_input, task_type, pipeline_position) + exclusion_hits = model_hard_exclusion_hits(user_input, model_rule) + if exclusion_hits: + item["score"] = 0.0 + item["hard_excluded"] = True + item["penalties"].append("model hard exclusion signal: " + "、".join(sorted(set(exclusion_hits))[:4])) + scored.append(item) no_call = all(item["score"] < threshold for item in scored) selected = [] rejected = [] @@ -160,7 +171,7 @@ def recommend(root, user_input, task_type="", pipeline_position="", threshold=No "score": item["score"], "reasons": item["reasons"], "penalties": item["penalties"], - "decision": "selected" if item["score"] >= threshold and not no_call else "rejected" + "decision": "selected" if item["score"] >= threshold and not no_call and not item.get("hard_excluded") else "rejected" } if output_item["decision"] == "selected": selected.append(output_item) diff --git a/selector/README.md b/selector/README.md index 89580d9..3cb4556 100644 --- a/selector/README.md +++ b/selector/README.md @@ -9,6 +9,7 @@ It should use simple matching rules: - Trigger keywords - Input types - Negative triggers +- Model-level hard exclusion triggers - Pipeline position - Selection priority - Negative trigger first @@ -18,6 +19,12 @@ It should use simple matching rules: - Direct-execution no-call signals - Calibration smoke checks +Round 03.2 selector rule: + +- Explicit refusal of a specific model, such as `不要思想考古` or `只做 QPI`, is a model-level hard exclusion for that model. +- A hard exclusion rejects only the named model; it does not globally no-call the request. +- Generic bare QPI gate words such as `如何` and `判断` should not be used alone. Use compound problem-definition phrases instead. + Current files: - `selector_rules.json` diff --git a/selector/selector_calibration_inputs.json b/selector/selector_calibration_inputs.json index a6042b6..1f7e5fe 100644 --- a/selector/selector_calibration_inputs.json +++ b/selector/selector_calibration_inputs.json @@ -591,6 +591,41 @@ "expected_selector_behavior": "no_call", "expected_notes": "Round 03.1 direct summary request must not select QPI by default score." }, + { + "case_id": "selector_calibration_ia_negative_explicit_do_not_call_001", + "category": "round_03_2_model_level_negative_gate", + "input": "QPI 已判断:这是 Issue。先不要思想考古,只做问题定性复核。", + "expected_selector_behavior": "select_qpi_reject_ia", + "expected_notes": "Explicit IA refusal must be a model-level hard exclusion, not a score penalty." + }, + { + "case_id": "selector_calibration_ia_negative_only_qpi_001", + "category": "round_03_2_model_level_negative_gate", + "input": "不要思想考古,只做 QPI 复核。", + "expected_selector_behavior": "select_qpi_reject_ia", + "expected_notes": "Only-QPI instruction should reject IA even when IA terms appear." + }, + { + "case_id": "selector_calibration_qpi_how_to_excel_no_call_001", + "category": "round_03_2_broad_qpi_gate_no_call", + "input": "如何在 Excel 里冻结首行?", + "expected_selector_behavior": "no_call", + "expected_notes": "Generic tool how-to should not be routed to QPI by bare 如何." + }, + { + "case_id": "selector_calibration_qpi_how_to_markdown_pdf_no_call_001", + "category": "round_03_2_broad_qpi_gate_no_call", + "input": "如何把 Markdown 转 PDF?", + "expected_selector_behavior": "no_call", + "expected_notes": "Generic conversion how-to should not be routed to QPI by bare 如何." + }, + { + "case_id": "selector_calibration_qpi_dictionary_judgment_no_call_001", + "category": "round_03_2_broad_qpi_gate_no_call", + "input": "判断这个词是什么意思。", + "expected_selector_behavior": "no_call", + "expected_notes": "Dictionary meaning request should not be routed to QPI by bare 判断." + }, { "case_id": "selector_calibration_analysis_override_should_call_001", "category": "round_03_1_analysis_override", diff --git a/selector/selector_rules.json b/selector/selector_rules.json index dc15608..faec365 100644 --- a/selector/selector_rules.json +++ b/selector/selector_rules.json @@ -79,8 +79,15 @@ "这是问题", "执行路径问题", "组织共识问题", - "如何", - "判断", + "如何判断这是什么问题", + "如何定义这个问题", + "如何提高流量", + "如何区分 Q/P/I", + "判断主导稀缺", + "判断这是问题还是课题", + "判断应该调用哪个模型", + "只做 QPI", + "只做问题定性", "真实调用", "上下文", "审计边界", @@ -92,6 +99,8 @@ "行动杠杆", "依赖风险", "误路由", + "真实独立 agent 调用记录", + "正式输出", "过度升维", "负激励", "成本转嫁", @@ -202,6 +211,14 @@ "先不要思想考古", "不要思想考古" ], + "hard_exclusion_triggers": [ + "不要思想考古", + "先不要思想考古", + "不要调用思想考古", + "不调用思想考古", + "只做 QPI", + "只做问题定性" + ], "pipeline_position": "modeling_depth_analysis", "selection_priority": 7, "routing_notes": "通常在 QPI 判断为中重型难题或课题后调用;遇到轻量事实检索应避免调用。" diff --git a/tests/intellectual_archaeology.regression.json b/tests/intellectual_archaeology.regression.json index 2c0c6eb..74cdeca 100644 --- a/tests/intellectual_archaeology.regression.json +++ b/tests/intellectual_archaeology.regression.json @@ -17,6 +17,8 @@ {"case_id": "case_ia_misuse_layer_jump_001", "model_id": "intellectual_archaeology", "case_type": "misuse", "input": "我们还没弄清应用层事实,就想直接讨论哲学基岩。", "expected_behavior": "应阻止层级跳跃,从应用层事实和证据缺口开始。", "failure_signal": "跳过中间层级。", "expected_output_elements": ["layers_to_analyze", "application"], "should_call_model": true, "expected_primary_model": "intellectual_archaeology", "negative_expected_models": [], "expected_classification": "not_applicable", "expected_dominant_scarcity": "not_applicable", "expected_max_depth": "application", "minimum_required_elements": ["stop_reason"], "forbidden_elements": ["philosophical_bedrock"], "evaluation_mode": "structured", "notes": "层级跳跃误用。"}, {"case_id": "case_ia_selector_gate_qpi_first_001", "model_id": "intellectual_archaeology", "case_type": "selector_gate", "input": "这个问题到底是缺数据、缺路径还是缺共识?", "expected_behavior": "selector 应先选 QPI,不应直接选思想考古。", "failure_signal": "IA 抢在 QPI 前。", "expected_output_elements": ["qpi first"], "should_call_model": false, "expected_primary_model": "qpi", "negative_expected_models": ["intellectual_archaeology"], "expected_classification": "not_applicable", "expected_dominant_scarcity": "not_applicable", "expected_max_depth": "no_call", "minimum_required_elements": ["rejected_models"], "forbidden_elements": ["selected IA"], "evaluation_mode": "structured", "notes": "QPI-before-IA gate。"}, {"case_id": "case_ia_selector_gate_no_heavy_signal_001", "model_id": "intellectual_archaeology", "case_type": "selector_gate", "input": "底层这个词怎么翻译?只要一个英文。", "expected_behavior": "虽然有底层一词,也不应召回思想考古。", "failure_signal": "关键词误召回。", "expected_output_elements": ["rejected_models"], "should_call_model": false, "expected_primary_model": "none", "negative_expected_models": ["intellectual_archaeology"], "expected_classification": "not_applicable", "expected_dominant_scarcity": "not_applicable", "expected_max_depth": "no_call", "minimum_required_elements": ["negative trigger"], "forbidden_elements": ["七层"], "evaluation_mode": "structured", "notes": "关键词误召回防护。"}, - {"case_id": "case_ia_pipeline_after_qpi_001", "model_id": "intellectual_archaeology", "case_type": "pipeline", "input": "QPI 已判断为中重型课题,且需要抽取底层机制和验证路径。", "expected_behavior": "IA 可作为 QPI 后续模型被调用。", "failure_signal": "在 QPI 放行后仍拒绝。", "expected_output_elements": ["QPI 后续", "should_call=true"], "should_call_model": true, "expected_primary_model": "intellectual_archaeology", "negative_expected_models": [], "expected_classification": "not_applicable", "expected_dominant_scarcity": "not_applicable", "expected_max_depth": "core_mechanism", "minimum_required_elements": ["entry_reason"], "forbidden_elements": [], "evaluation_mode": "structured", "notes": "QPI 后续 pipeline。"} + {"case_id": "case_ia_pipeline_after_qpi_001", "model_id": "intellectual_archaeology", "case_type": "pipeline", "input": "QPI 已判断为中重型课题,且需要抽取底层机制和验证路径。", "expected_behavior": "IA 可作为 QPI 后续模型被调用。", "failure_signal": "在 QPI 放行后仍拒绝。", "expected_output_elements": ["QPI 后续", "should_call=true"], "should_call_model": true, "expected_primary_model": "intellectual_archaeology", "negative_expected_models": [], "expected_classification": "not_applicable", "expected_dominant_scarcity": "not_applicable", "expected_max_depth": "core_mechanism", "minimum_required_elements": ["entry_reason"], "forbidden_elements": [], "evaluation_mode": "structured", "notes": "QPI 后续 pipeline。"}, + {"case_id": "case_ia_negative_explicit_do_not_call_001", "model_id": "intellectual_archaeology", "case_type": "selector_gate", "input": "QPI 已判断:这是 Issue。先不要思想考古,只做问题定性复核。", "expected_behavior": "selector 应选择 QPI,并硬拒绝思想考古。", "failure_signal": "因为“思想考古”和“QPI 已判断”加分而仍选中 IA。", "expected_output_elements": ["selected=qpi", "rejected=intellectual_archaeology"], "should_call_model": false, "expected_primary_model": "qpi", "negative_expected_models": ["intellectual_archaeology"], "expected_classification": "not_applicable", "expected_dominant_scarcity": "not_applicable", "expected_max_depth": "no_call", "minimum_required_elements": ["model_level_hard_exclusion"], "forbidden_elements": ["selected IA"], "evaluation_mode": "structured", "notes": "Round 03.2 hard exclusion trap: explicit IA refusal must override score."}, + {"case_id": "case_ia_negative_only_qpi_001", "model_id": "intellectual_archaeology", "case_type": "selector_gate", "input": "不要思想考古,只做 QPI 复核。", "expected_behavior": "selector 应选择 QPI,并硬拒绝思想考古。", "failure_signal": "IA 负触发只被当作 penalty,仍可能被选中。", "expected_output_elements": ["selected=qpi", "rejected=intellectual_archaeology"], "should_call_model": false, "expected_primary_model": "qpi", "negative_expected_models": ["intellectual_archaeology"], "expected_classification": "not_applicable", "expected_dominant_scarcity": "not_applicable", "expected_max_depth": "no_call", "minimum_required_elements": ["model_level_hard_exclusion"], "forbidden_elements": ["selected IA"], "evaluation_mode": "structured", "notes": "Round 03.2 hard exclusion trap: only-QPI instruction rejects IA."} ] } diff --git a/tests/qpi.regression.json b/tests/qpi.regression.json index f61f495..4b3a6ba 100644 --- a/tests/qpi.regression.json +++ b/tests/qpi.regression.json @@ -1550,6 +1550,90 @@ "evaluation_mode": "structured", "notes": "Round 03.1 direct execution no-call." }, + { + "case_id": "case_qpi_how_to_excel_no_call_001", + "model_id": "qpi", + "case_type": "no_call", + "input": "如何在 Excel 里冻结首行?", + "expected_behavior": "应 no_call:普通工具 how-to 不应因裸“如何”触发 QPI。", + "failure_signal": "selector 选中 QPI。", + "expected_output_elements": [ + "classification=no_call" + ], + "should_call_model": false, + "expected_primary_model": "none", + "negative_expected_models": [ + "qpi", + "intellectual_archaeology" + ], + "expected_classification": "no_call", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "not_applicable", + "minimum_required_elements": [ + "no_call" + ], + "forbidden_elements": [ + "problem_definition" + ], + "evaluation_mode": "structured", + "notes": "Round 03.2 no-call trap: narrow bare qpi_gate_signals." + }, + { + "case_id": "case_qpi_how_to_markdown_pdf_no_call_001", + "model_id": "qpi", + "case_type": "no_call", + "input": "如何把 Markdown 转 PDF?", + "expected_behavior": "应 no_call:普通格式转换 how-to 不应调用 QPI。", + "failure_signal": "selector 选中 QPI。", + "expected_output_elements": [ + "classification=no_call" + ], + "should_call_model": false, + "expected_primary_model": "none", + "negative_expected_models": [ + "qpi", + "intellectual_archaeology" + ], + "expected_classification": "no_call", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "not_applicable", + "minimum_required_elements": [ + "no_call" + ], + "forbidden_elements": [ + "problem_definition" + ], + "evaluation_mode": "structured", + "notes": "Round 03.2 no-call trap: generic how-to is not QPI." + }, + { + "case_id": "case_qpi_dictionary_judgment_no_call_001", + "model_id": "qpi", + "case_type": "no_call", + "input": "判断这个词是什么意思。", + "expected_behavior": "应 no_call:普通词义判断不应因裸“判断”触发 QPI。", + "failure_signal": "selector 选中 QPI。", + "expected_output_elements": [ + "classification=no_call" + ], + "should_call_model": false, + "expected_primary_model": "none", + "negative_expected_models": [ + "qpi", + "intellectual_archaeology" + ], + "expected_classification": "no_call", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "not_applicable", + "minimum_required_elements": [ + "no_call" + ], + "forbidden_elements": [ + "problem_definition" + ], + "evaluation_mode": "structured", + "notes": "Round 03.2 no-call trap: dictionary judgment is not QPI." + }, { "case_id": "case_qpi_analysis_override_should_call_001", "model_id": "qpi", diff --git a/tests/regression_cases.json b/tests/regression_cases.json index 5b5ae6b..dc74827 100644 --- a/tests/regression_cases.json +++ b/tests/regression_cases.json @@ -644,447 +644,6 @@ "source_case_digest_id": "qpi-disappointment-008", "notes": "Owner-reviewed disappointment case 008. Canonical anchor: one-person self-governance can still be Issue." }, - { - "case_id": "case_ia_positive_deep_modeling_001", - "model_id": "intellectual_archaeology", - "case_type": "positive", - "input": "我们要把专家对课堂质量的直觉变成一个可复用评价模型,但不知道底层教育目的和学习机制如何支撑指标。", - "expected_behavior": "应调用思想考古学,下潜到核心机理层,标出验证需求。", - "failure_signal": "只总结表层指标。", - "expected_output_elements": [ - "should_call=true", - "recommended_max_depth=core_mechanism" - ], - "should_call_model": true, - "expected_primary_model": "intellectual_archaeology", - "negative_expected_models": [], - "expected_classification": "not_applicable", - "expected_dominant_scarcity": "not_applicable", - "expected_max_depth": "core_mechanism", - "minimum_required_elements": [ - "stop_reason", - "validation_needed" - ], - "forbidden_elements": [], - "evaluation_mode": "structured", - "notes": "重型正例。" - }, - { - "case_id": "case_ia_positive_resilience_001", - "model_id": "intellectual_archaeology", - "case_type": "positive", - "input": "新的教学工具出现后,我们想知道现有评价模型需要局部调整还是必须重建底层教育模型。", - "expected_behavior": "应区分稳定内核和可变外壳,判断冲击层级。", - "failure_signal": "动辄推倒全部模型。", - "expected_output_elements": [ - "稳定内核", - "冲击层级" - ], - "should_call_model": true, - "expected_primary_model": "intellectual_archaeology", - "negative_expected_models": [], - "expected_classification": "not_applicable", - "expected_dominant_scarcity": "not_applicable", - "expected_max_depth": "purpose", - "minimum_required_elements": [ - "layers_to_analyze", - "action_implication" - ], - "forbidden_elements": [ - "默认哲学基岩" - ], - "evaluation_mode": "structured", - "notes": "中重型正例。" - }, - { - "case_id": "case_ia_positive_model_extraction_001", - "model_id": "intellectual_archaeology", - "case_type": "positive", - "input": "我要从一篇长文章中抽取可复用认知模型,需要判断它的机制、边界和底层假设。", - "expected_behavior": "应调用思想考古,最多到核心机理层或人类能力层。", - "failure_signal": "只做摘要或直接上哲学。", - "expected_output_elements": [ - "模型抽取", - "recommended_max_depth" - ], - "should_call_model": true, - "expected_primary_model": "intellectual_archaeology", - "negative_expected_models": [], - "expected_classification": "not_applicable", - "expected_dominant_scarcity": "not_applicable", - "expected_max_depth": "core_mechanism", - "minimum_required_elements": [ - "assumptions_by_layer", - "validation_needed" - ], - "forbidden_elements": [], - "evaluation_mode": "structured", - "notes": "模型抽取场景。" - }, - { - "case_id": "case_ia_positive_heavy_failure_001", - "model_id": "intellectual_archaeology", - "case_type": "positive", - "input": "这个战略方案连续三次失败,但每次复盘都只停留在执行动作,我们怀疑底层目的和机制错了。", - "expected_behavior": "应调用,允许下潜到目的层或核心机理层。", - "failure_signal": "只给执行建议。", - "expected_output_elements": [ - "entry_reason", - "core_mechanism" - ], - "should_call_model": true, - "expected_primary_model": "intellectual_archaeology", - "negative_expected_models": [], - "expected_classification": "not_applicable", - "expected_dominant_scarcity": "not_applicable", - "expected_max_depth": "core_mechanism", - "minimum_required_elements": [ - "no_deeper_reason" - ], - "forbidden_elements": [], - "evaluation_mode": "structured", - "notes": "重型失败复盘。" - }, - { - "case_id": "case_ia_positive_heavy_philosophy_001", - "model_id": "intellectual_archaeology", - "case_type": "positive", - "input": "我们正在定义问题本身的模型,涉及认知主体和现实映射的关系。", - "expected_behavior": "可下潜到哲学基岩层,但必须说明为何需要。", - "failure_signal": "不说明理由就上哲学。", - "expected_output_elements": [ - "philosophical_bedrock", - "entry_reason" - ], - "should_call_model": true, - "expected_primary_model": "intellectual_archaeology", - "negative_expected_models": [], - "expected_classification": "not_applicable", - "expected_dominant_scarcity": "not_applicable", - "expected_max_depth": "philosophical_bedrock", - "minimum_required_elements": [ - "stop_reason", - "action_implication" - ], - "forbidden_elements": [], - "evaluation_mode": "structured", - "notes": "少数可到哲学基岩的正例。" - }, - { - "case_id": "case_ia_boundary_medium_purpose_001", - "model_id": "intellectual_archaeology", - "case_type": "boundary", - "input": "我们知道课堂评价指标,但不知道它服务的教育目的是否一致。", - "expected_behavior": "应下潜到目的层后停止。", - "failure_signal": "自动下潜到哲学基岩。", - "expected_output_elements": [ - "recommended_max_depth=purpose" - ], - "should_call_model": true, - "expected_primary_model": "intellectual_archaeology", - "negative_expected_models": [], - "expected_classification": "not_applicable", - "expected_dominant_scarcity": "not_applicable", - "expected_max_depth": "purpose", - "minimum_required_elements": [ - "no_deeper_reason" - ], - "forbidden_elements": [ - "philosophical_bedrock" - ], - "evaluation_mode": "structured", - "notes": "中型目的层。" - }, - { - "case_id": "case_ia_boundary_medium_process_001", - "model_id": "intellectual_archaeology", - "case_type": "boundary", - "input": "流程卡住了,我们需要理解实际业务过程,而不是重写价值观。", - "expected_behavior": "最多到过程层,说明停止理由。", - "failure_signal": "展开人类能力或哲学层。", - "expected_output_elements": [ - "recommended_max_depth=process" - ], - "should_call_model": true, - "expected_primary_model": "intellectual_archaeology", - "negative_expected_models": [], - "expected_classification": "not_applicable", - "expected_dominant_scarcity": "not_applicable", - "expected_max_depth": "process", - "minimum_required_elements": [ - "stop_reason" - ], - "forbidden_elements": [ - "human_capability", - "philosophical_bedrock" - ], - "evaluation_mode": "structured", - "notes": "过程层停止。" - }, - { - "case_id": "case_ia_boundary_medium_core_001", - "model_id": "intellectual_archaeology", - "case_type": "boundary", - "input": "我们已经有流程图,但不知道学习机制是否支撑这个流程。", - "expected_behavior": "可下潜到核心机理层,不应继续到哲学基岩。", - "failure_signal": "无条件七层全开。", - "expected_output_elements": [ - "core_mechanism", - "no_deeper_reason" - ], - "should_call_model": true, - "expected_primary_model": "intellectual_archaeology", - "negative_expected_models": [], - "expected_classification": "not_applicable", - "expected_dominant_scarcity": "not_applicable", - "expected_max_depth": "core_mechanism", - "minimum_required_elements": [ - "validation_needed" - ], - "forbidden_elements": [ - "philosophical_bedrock" - ], - "evaluation_mode": "structured", - "notes": "核心机理层停止。" - }, - { - "case_id": "case_ia_no_call_fact_001", - "model_id": "intellectual_archaeology", - "case_type": "no_call", - "input": "我只是想知道这篇文章里有没有提到某个工具名称。", - "expected_behavior": "不应调用思想考古,应建议直接检索。", - "failure_signal": "强行展开七层哲学分析。", - "expected_output_elements": [ - "should_call=false" - ], - "should_call_model": false, - "expected_primary_model": "none", - "negative_expected_models": [ - "intellectual_archaeology" - ], - "expected_classification": "not_applicable", - "expected_dominant_scarcity": "not_applicable", - "expected_max_depth": "no_call", - "minimum_required_elements": [ - "entry_reason" - ], - "forbidden_elements": [ - "七层" - ], - "evaluation_mode": "structured", - "notes": "事实检索 no-call。" - }, - { - "case_id": "case_ia_no_call_edit_001", - "model_id": "intellectual_archaeology", - "case_type": "no_call", - "input": "请把这段话润色一下,不要深入分析。", - "expected_behavior": "不调用。", - "failure_signal": "展开底层假设。", - "expected_output_elements": [ - "should_call=false" - ], - "should_call_model": false, - "expected_primary_model": "none", - "negative_expected_models": [ - "intellectual_archaeology" - ], - "expected_classification": "not_applicable", - "expected_dominant_scarcity": "not_applicable", - "expected_max_depth": "no_call", - "minimum_required_elements": [ - "no_deeper_reason" - ], - "forbidden_elements": [ - "底层假设" - ], - "evaluation_mode": "structured", - "notes": "润色 no-call。" - }, - { - "case_id": "case_ia_no_call_light_execution_001", - "model_id": "intellectual_archaeology", - "case_type": "no_call", - "input": "按钮文案写错了,直接改成保存即可。", - "expected_behavior": "不调用。", - "failure_signal": "重新定义用户关系。", - "expected_output_elements": [ - "should_call=false" - ], - "should_call_model": false, - "expected_primary_model": "none", - "negative_expected_models": [ - "intellectual_archaeology" - ], - "expected_classification": "not_applicable", - "expected_dominant_scarcity": "not_applicable", - "expected_max_depth": "no_call", - "minimum_required_elements": [ - "action_implication" - ], - "forbidden_elements": [ - "philosophical_bedrock" - ], - "evaluation_mode": "structured", - "notes": "轻量执行 no-call。" - }, - { - "case_id": "case_ia_misuse_philosophizing_001", - "model_id": "intellectual_archaeology", - "case_type": "misuse", - "input": "我们已经知道方案只需要修一个按钮文案,但还想顺便从哲学基岩重新定义用户关系。", - "expected_behavior": "应拒绝过度下钻,说明继续下潜不会改变判断或行动。", - "failure_signal": "接受无必要的哲学化扩展。", - "expected_output_elements": [ - "停止下钻", - "行动不改变" - ], - "should_call_model": false, - "expected_primary_model": "none", - "negative_expected_models": [ - "intellectual_archaeology" - ], - "expected_classification": "not_applicable", - "expected_dominant_scarcity": "not_applicable", - "expected_max_depth": "no_call", - "minimum_required_elements": [ - "no_deeper_reason" - ], - "forbidden_elements": [ - "philosophical_bedrock" - ], - "evaluation_mode": "structured", - "notes": "过度哲学化。" - }, - { - "case_id": "case_ia_misuse_no_validation_001", - "model_id": "intellectual_archaeology", - "case_type": "misuse", - "input": "我们用七层结构写出一个很自洽的模型,准备直接上线,不做红队、数据对撞或专家盲审。", - "expected_behavior": "应指出内部自洽不等于现实有效,要求补充验证机制。", - "failure_signal": "把七层结构本身视为充分验证。", - "expected_output_elements": [ - "validation_needed" - ], - "should_call_model": true, - "expected_primary_model": "intellectual_archaeology", - "negative_expected_models": [], - "expected_classification": "not_applicable", - "expected_dominant_scarcity": "not_applicable", - "expected_max_depth": "core_mechanism", - "minimum_required_elements": [ - "validation_needed" - ], - "forbidden_elements": [ - "直接上线" - ], - "evaluation_mode": "structured", - "notes": "无验证误用。" - }, - { - "case_id": "case_ia_misuse_layer_jump_001", - "model_id": "intellectual_archaeology", - "case_type": "misuse", - "input": "我们还没弄清应用层事实,就想直接讨论哲学基岩。", - "expected_behavior": "应阻止层级跳跃,从应用层事实和证据缺口开始。", - "failure_signal": "跳过中间层级。", - "expected_output_elements": [ - "layers_to_analyze", - "application" - ], - "should_call_model": true, - "expected_primary_model": "intellectual_archaeology", - "negative_expected_models": [], - "expected_classification": "not_applicable", - "expected_dominant_scarcity": "not_applicable", - "expected_max_depth": "application", - "minimum_required_elements": [ - "stop_reason" - ], - "forbidden_elements": [ - "philosophical_bedrock" - ], - "evaluation_mode": "structured", - "notes": "层级跳跃误用。" - }, - { - "case_id": "case_ia_selector_gate_qpi_first_001", - "model_id": "intellectual_archaeology", - "case_type": "selector_gate", - "input": "这个问题到底是缺数据、缺路径还是缺共识?", - "expected_behavior": "selector 应先选 QPI,不应直接选思想考古。", - "failure_signal": "IA 抢在 QPI 前。", - "expected_output_elements": [ - "qpi first" - ], - "should_call_model": false, - "expected_primary_model": "qpi", - "negative_expected_models": [ - "intellectual_archaeology" - ], - "expected_classification": "not_applicable", - "expected_dominant_scarcity": "not_applicable", - "expected_max_depth": "no_call", - "minimum_required_elements": [ - "rejected_models" - ], - "forbidden_elements": [ - "selected IA" - ], - "evaluation_mode": "structured", - "notes": "QPI-before-IA gate。" - }, - { - "case_id": "case_ia_selector_gate_no_heavy_signal_001", - "model_id": "intellectual_archaeology", - "case_type": "selector_gate", - "input": "底层这个词怎么翻译?只要一个英文。", - "expected_behavior": "虽然有底层一词,也不应召回思想考古。", - "failure_signal": "关键词误召回。", - "expected_output_elements": [ - "rejected_models" - ], - "should_call_model": false, - "expected_primary_model": "none", - "negative_expected_models": [ - "intellectual_archaeology" - ], - "expected_classification": "not_applicable", - "expected_dominant_scarcity": "not_applicable", - "expected_max_depth": "no_call", - "minimum_required_elements": [ - "negative trigger" - ], - "forbidden_elements": [ - "七层" - ], - "evaluation_mode": "structured", - "notes": "关键词误召回防护。" - }, - { - "case_id": "case_ia_pipeline_after_qpi_001", - "model_id": "intellectual_archaeology", - "case_type": "pipeline", - "input": "QPI 已判断为中重型课题,且需要抽取底层机制和验证路径。", - "expected_behavior": "IA 可作为 QPI 后续模型被调用。", - "failure_signal": "在 QPI 放行后仍拒绝。", - "expected_output_elements": [ - "QPI 后续", - "should_call=true" - ], - "should_call_model": true, - "expected_primary_model": "intellectual_archaeology", - "negative_expected_models": [], - "expected_classification": "not_applicable", - "expected_dominant_scarcity": "not_applicable", - "expected_max_depth": "core_mechanism", - "minimum_required_elements": [ - "entry_reason" - ], - "forbidden_elements": [], - "evaluation_mode": "structured", - "notes": "QPI 后续 pipeline。" - }, { "case_id": "case_qpi_org_hard_resource_documentation_001", "model_id": "qpi", @@ -1990,6 +1549,90 @@ "evaluation_mode": "structured", "notes": "Round 03.1 direct execution no-call." }, + { + "case_id": "case_qpi_how_to_excel_no_call_001", + "model_id": "qpi", + "case_type": "no_call", + "input": "如何在 Excel 里冻结首行?", + "expected_behavior": "应 no_call:普通工具 how-to 不应因裸“如何”触发 QPI。", + "failure_signal": "selector 选中 QPI。", + "expected_output_elements": [ + "classification=no_call" + ], + "should_call_model": false, + "expected_primary_model": "none", + "negative_expected_models": [ + "qpi", + "intellectual_archaeology" + ], + "expected_classification": "no_call", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "not_applicable", + "minimum_required_elements": [ + "no_call" + ], + "forbidden_elements": [ + "problem_definition" + ], + "evaluation_mode": "structured", + "notes": "Round 03.2 no-call trap: narrow bare qpi_gate_signals." + }, + { + "case_id": "case_qpi_how_to_markdown_pdf_no_call_001", + "model_id": "qpi", + "case_type": "no_call", + "input": "如何把 Markdown 转 PDF?", + "expected_behavior": "应 no_call:普通格式转换 how-to 不应调用 QPI。", + "failure_signal": "selector 选中 QPI。", + "expected_output_elements": [ + "classification=no_call" + ], + "should_call_model": false, + "expected_primary_model": "none", + "negative_expected_models": [ + "qpi", + "intellectual_archaeology" + ], + "expected_classification": "no_call", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "not_applicable", + "minimum_required_elements": [ + "no_call" + ], + "forbidden_elements": [ + "problem_definition" + ], + "evaluation_mode": "structured", + "notes": "Round 03.2 no-call trap: generic how-to is not QPI." + }, + { + "case_id": "case_qpi_dictionary_judgment_no_call_001", + "model_id": "qpi", + "case_type": "no_call", + "input": "判断这个词是什么意思。", + "expected_behavior": "应 no_call:普通词义判断不应因裸“判断”触发 QPI。", + "failure_signal": "selector 选中 QPI。", + "expected_output_elements": [ + "classification=no_call" + ], + "should_call_model": false, + "expected_primary_model": "none", + "negative_expected_models": [ + "qpi", + "intellectual_archaeology" + ], + "expected_classification": "no_call", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "not_applicable", + "minimum_required_elements": [ + "no_call" + ], + "forbidden_elements": [ + "problem_definition" + ], + "evaluation_mode": "structured", + "notes": "Round 03.2 no-call trap: dictionary judgment is not QPI." + }, { "case_id": "case_qpi_analysis_override_should_call_001", "model_id": "qpi", @@ -2017,6 +1660,503 @@ ], "evaluation_mode": "structured", "notes": "Round 03.1 override trap: direct execution phrase plus explicit analysis request." + }, + { + "case_id": "case_ia_positive_deep_modeling_001", + "model_id": "intellectual_archaeology", + "case_type": "positive", + "input": "我们要把专家对课堂质量的直觉变成一个可复用评价模型,但不知道底层教育目的和学习机制如何支撑指标。", + "expected_behavior": "应调用思想考古学,下潜到核心机理层,标出验证需求。", + "failure_signal": "只总结表层指标。", + "expected_output_elements": [ + "should_call=true", + "recommended_max_depth=core_mechanism" + ], + "should_call_model": true, + "expected_primary_model": "intellectual_archaeology", + "negative_expected_models": [], + "expected_classification": "not_applicable", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "core_mechanism", + "minimum_required_elements": [ + "stop_reason", + "validation_needed" + ], + "forbidden_elements": [], + "evaluation_mode": "structured", + "notes": "重型正例。" + }, + { + "case_id": "case_ia_positive_resilience_001", + "model_id": "intellectual_archaeology", + "case_type": "positive", + "input": "新的教学工具出现后,我们想知道现有评价模型需要局部调整还是必须重建底层教育模型。", + "expected_behavior": "应区分稳定内核和可变外壳,判断冲击层级。", + "failure_signal": "动辄推倒全部模型。", + "expected_output_elements": [ + "稳定内核", + "冲击层级" + ], + "should_call_model": true, + "expected_primary_model": "intellectual_archaeology", + "negative_expected_models": [], + "expected_classification": "not_applicable", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "purpose", + "minimum_required_elements": [ + "layers_to_analyze", + "action_implication" + ], + "forbidden_elements": [ + "默认哲学基岩" + ], + "evaluation_mode": "structured", + "notes": "中重型正例。" + }, + { + "case_id": "case_ia_positive_model_extraction_001", + "model_id": "intellectual_archaeology", + "case_type": "positive", + "input": "我要从一篇长文章中抽取可复用认知模型,需要判断它的机制、边界和底层假设。", + "expected_behavior": "应调用思想考古,最多到核心机理层或人类能力层。", + "failure_signal": "只做摘要或直接上哲学。", + "expected_output_elements": [ + "模型抽取", + "recommended_max_depth" + ], + "should_call_model": true, + "expected_primary_model": "intellectual_archaeology", + "negative_expected_models": [], + "expected_classification": "not_applicable", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "core_mechanism", + "minimum_required_elements": [ + "assumptions_by_layer", + "validation_needed" + ], + "forbidden_elements": [], + "evaluation_mode": "structured", + "notes": "模型抽取场景。" + }, + { + "case_id": "case_ia_positive_heavy_failure_001", + "model_id": "intellectual_archaeology", + "case_type": "positive", + "input": "这个战略方案连续三次失败,但每次复盘都只停留在执行动作,我们怀疑底层目的和机制错了。", + "expected_behavior": "应调用,允许下潜到目的层或核心机理层。", + "failure_signal": "只给执行建议。", + "expected_output_elements": [ + "entry_reason", + "core_mechanism" + ], + "should_call_model": true, + "expected_primary_model": "intellectual_archaeology", + "negative_expected_models": [], + "expected_classification": "not_applicable", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "core_mechanism", + "minimum_required_elements": [ + "no_deeper_reason" + ], + "forbidden_elements": [], + "evaluation_mode": "structured", + "notes": "重型失败复盘。" + }, + { + "case_id": "case_ia_positive_heavy_philosophy_001", + "model_id": "intellectual_archaeology", + "case_type": "positive", + "input": "我们正在定义问题本身的模型,涉及认知主体和现实映射的关系。", + "expected_behavior": "可下潜到哲学基岩层,但必须说明为何需要。", + "failure_signal": "不说明理由就上哲学。", + "expected_output_elements": [ + "philosophical_bedrock", + "entry_reason" + ], + "should_call_model": true, + "expected_primary_model": "intellectual_archaeology", + "negative_expected_models": [], + "expected_classification": "not_applicable", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "philosophical_bedrock", + "minimum_required_elements": [ + "stop_reason", + "action_implication" + ], + "forbidden_elements": [], + "evaluation_mode": "structured", + "notes": "少数可到哲学基岩的正例。" + }, + { + "case_id": "case_ia_boundary_medium_purpose_001", + "model_id": "intellectual_archaeology", + "case_type": "boundary", + "input": "我们知道课堂评价指标,但不知道它服务的教育目的是否一致。", + "expected_behavior": "应下潜到目的层后停止。", + "failure_signal": "自动下潜到哲学基岩。", + "expected_output_elements": [ + "recommended_max_depth=purpose" + ], + "should_call_model": true, + "expected_primary_model": "intellectual_archaeology", + "negative_expected_models": [], + "expected_classification": "not_applicable", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "purpose", + "minimum_required_elements": [ + "no_deeper_reason" + ], + "forbidden_elements": [ + "philosophical_bedrock" + ], + "evaluation_mode": "structured", + "notes": "中型目的层。" + }, + { + "case_id": "case_ia_boundary_medium_process_001", + "model_id": "intellectual_archaeology", + "case_type": "boundary", + "input": "流程卡住了,我们需要理解实际业务过程,而不是重写价值观。", + "expected_behavior": "最多到过程层,说明停止理由。", + "failure_signal": "展开人类能力或哲学层。", + "expected_output_elements": [ + "recommended_max_depth=process" + ], + "should_call_model": true, + "expected_primary_model": "intellectual_archaeology", + "negative_expected_models": [], + "expected_classification": "not_applicable", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "process", + "minimum_required_elements": [ + "stop_reason" + ], + "forbidden_elements": [ + "human_capability", + "philosophical_bedrock" + ], + "evaluation_mode": "structured", + "notes": "过程层停止。" + }, + { + "case_id": "case_ia_boundary_medium_core_001", + "model_id": "intellectual_archaeology", + "case_type": "boundary", + "input": "我们已经有流程图,但不知道学习机制是否支撑这个流程。", + "expected_behavior": "可下潜到核心机理层,不应继续到哲学基岩。", + "failure_signal": "无条件七层全开。", + "expected_output_elements": [ + "core_mechanism", + "no_deeper_reason" + ], + "should_call_model": true, + "expected_primary_model": "intellectual_archaeology", + "negative_expected_models": [], + "expected_classification": "not_applicable", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "core_mechanism", + "minimum_required_elements": [ + "validation_needed" + ], + "forbidden_elements": [ + "philosophical_bedrock" + ], + "evaluation_mode": "structured", + "notes": "核心机理层停止。" + }, + { + "case_id": "case_ia_no_call_fact_001", + "model_id": "intellectual_archaeology", + "case_type": "no_call", + "input": "我只是想知道这篇文章里有没有提到某个工具名称。", + "expected_behavior": "不应调用思想考古,应建议直接检索。", + "failure_signal": "强行展开七层哲学分析。", + "expected_output_elements": [ + "should_call=false" + ], + "should_call_model": false, + "expected_primary_model": "none", + "negative_expected_models": [ + "intellectual_archaeology" + ], + "expected_classification": "not_applicable", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "no_call", + "minimum_required_elements": [ + "entry_reason" + ], + "forbidden_elements": [ + "七层" + ], + "evaluation_mode": "structured", + "notes": "事实检索 no-call。" + }, + { + "case_id": "case_ia_no_call_edit_001", + "model_id": "intellectual_archaeology", + "case_type": "no_call", + "input": "请把这段话润色一下,不要深入分析。", + "expected_behavior": "不调用。", + "failure_signal": "展开底层假设。", + "expected_output_elements": [ + "should_call=false" + ], + "should_call_model": false, + "expected_primary_model": "none", + "negative_expected_models": [ + "intellectual_archaeology" + ], + "expected_classification": "not_applicable", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "no_call", + "minimum_required_elements": [ + "no_deeper_reason" + ], + "forbidden_elements": [ + "底层假设" + ], + "evaluation_mode": "structured", + "notes": "润色 no-call。" + }, + { + "case_id": "case_ia_no_call_light_execution_001", + "model_id": "intellectual_archaeology", + "case_type": "no_call", + "input": "按钮文案写错了,直接改成保存即可。", + "expected_behavior": "不调用。", + "failure_signal": "重新定义用户关系。", + "expected_output_elements": [ + "should_call=false" + ], + "should_call_model": false, + "expected_primary_model": "none", + "negative_expected_models": [ + "intellectual_archaeology" + ], + "expected_classification": "not_applicable", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "no_call", + "minimum_required_elements": [ + "action_implication" + ], + "forbidden_elements": [ + "philosophical_bedrock" + ], + "evaluation_mode": "structured", + "notes": "轻量执行 no-call。" + }, + { + "case_id": "case_ia_misuse_philosophizing_001", + "model_id": "intellectual_archaeology", + "case_type": "misuse", + "input": "我们已经知道方案只需要修一个按钮文案,但还想顺便从哲学基岩重新定义用户关系。", + "expected_behavior": "应拒绝过度下钻,说明继续下潜不会改变判断或行动。", + "failure_signal": "接受无必要的哲学化扩展。", + "expected_output_elements": [ + "停止下钻", + "行动不改变" + ], + "should_call_model": false, + "expected_primary_model": "none", + "negative_expected_models": [ + "intellectual_archaeology" + ], + "expected_classification": "not_applicable", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "no_call", + "minimum_required_elements": [ + "no_deeper_reason" + ], + "forbidden_elements": [ + "philosophical_bedrock" + ], + "evaluation_mode": "structured", + "notes": "过度哲学化。" + }, + { + "case_id": "case_ia_misuse_no_validation_001", + "model_id": "intellectual_archaeology", + "case_type": "misuse", + "input": "我们用七层结构写出一个很自洽的模型,准备直接上线,不做红队、数据对撞或专家盲审。", + "expected_behavior": "应指出内部自洽不等于现实有效,要求补充验证机制。", + "failure_signal": "把七层结构本身视为充分验证。", + "expected_output_elements": [ + "validation_needed" + ], + "should_call_model": true, + "expected_primary_model": "intellectual_archaeology", + "negative_expected_models": [], + "expected_classification": "not_applicable", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "core_mechanism", + "minimum_required_elements": [ + "validation_needed" + ], + "forbidden_elements": [ + "直接上线" + ], + "evaluation_mode": "structured", + "notes": "无验证误用。" + }, + { + "case_id": "case_ia_misuse_layer_jump_001", + "model_id": "intellectual_archaeology", + "case_type": "misuse", + "input": "我们还没弄清应用层事实,就想直接讨论哲学基岩。", + "expected_behavior": "应阻止层级跳跃,从应用层事实和证据缺口开始。", + "failure_signal": "跳过中间层级。", + "expected_output_elements": [ + "layers_to_analyze", + "application" + ], + "should_call_model": true, + "expected_primary_model": "intellectual_archaeology", + "negative_expected_models": [], + "expected_classification": "not_applicable", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "application", + "minimum_required_elements": [ + "stop_reason" + ], + "forbidden_elements": [ + "philosophical_bedrock" + ], + "evaluation_mode": "structured", + "notes": "层级跳跃误用。" + }, + { + "case_id": "case_ia_selector_gate_qpi_first_001", + "model_id": "intellectual_archaeology", + "case_type": "selector_gate", + "input": "这个问题到底是缺数据、缺路径还是缺共识?", + "expected_behavior": "selector 应先选 QPI,不应直接选思想考古。", + "failure_signal": "IA 抢在 QPI 前。", + "expected_output_elements": [ + "qpi first" + ], + "should_call_model": false, + "expected_primary_model": "qpi", + "negative_expected_models": [ + "intellectual_archaeology" + ], + "expected_classification": "not_applicable", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "no_call", + "minimum_required_elements": [ + "rejected_models" + ], + "forbidden_elements": [ + "selected IA" + ], + "evaluation_mode": "structured", + "notes": "QPI-before-IA gate。" + }, + { + "case_id": "case_ia_selector_gate_no_heavy_signal_001", + "model_id": "intellectual_archaeology", + "case_type": "selector_gate", + "input": "底层这个词怎么翻译?只要一个英文。", + "expected_behavior": "虽然有底层一词,也不应召回思想考古。", + "failure_signal": "关键词误召回。", + "expected_output_elements": [ + "rejected_models" + ], + "should_call_model": false, + "expected_primary_model": "none", + "negative_expected_models": [ + "intellectual_archaeology" + ], + "expected_classification": "not_applicable", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "no_call", + "minimum_required_elements": [ + "negative trigger" + ], + "forbidden_elements": [ + "七层" + ], + "evaluation_mode": "structured", + "notes": "关键词误召回防护。" + }, + { + "case_id": "case_ia_pipeline_after_qpi_001", + "model_id": "intellectual_archaeology", + "case_type": "pipeline", + "input": "QPI 已判断为中重型课题,且需要抽取底层机制和验证路径。", + "expected_behavior": "IA 可作为 QPI 后续模型被调用。", + "failure_signal": "在 QPI 放行后仍拒绝。", + "expected_output_elements": [ + "QPI 后续", + "should_call=true" + ], + "should_call_model": true, + "expected_primary_model": "intellectual_archaeology", + "negative_expected_models": [], + "expected_classification": "not_applicable", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "core_mechanism", + "minimum_required_elements": [ + "entry_reason" + ], + "forbidden_elements": [], + "evaluation_mode": "structured", + "notes": "QPI 后续 pipeline。" + }, + { + "case_id": "case_ia_negative_explicit_do_not_call_001", + "model_id": "intellectual_archaeology", + "case_type": "selector_gate", + "input": "QPI 已判断:这是 Issue。先不要思想考古,只做问题定性复核。", + "expected_behavior": "selector 应选择 QPI,并硬拒绝思想考古。", + "failure_signal": "因为“思想考古”和“QPI 已判断”加分而仍选中 IA。", + "expected_output_elements": [ + "selected=qpi", + "rejected=intellectual_archaeology" + ], + "should_call_model": false, + "expected_primary_model": "qpi", + "negative_expected_models": [ + "intellectual_archaeology" + ], + "expected_classification": "not_applicable", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "no_call", + "minimum_required_elements": [ + "model_level_hard_exclusion" + ], + "forbidden_elements": [ + "selected IA" + ], + "evaluation_mode": "structured", + "notes": "Round 03.2 hard exclusion trap: explicit IA refusal must override score." + }, + { + "case_id": "case_ia_negative_only_qpi_001", + "model_id": "intellectual_archaeology", + "case_type": "selector_gate", + "input": "不要思想考古,只做 QPI 复核。", + "expected_behavior": "selector 应选择 QPI,并硬拒绝思想考古。", + "failure_signal": "IA 负触发只被当作 penalty,仍可能被选中。", + "expected_output_elements": [ + "selected=qpi", + "rejected=intellectual_archaeology" + ], + "should_call_model": false, + "expected_primary_model": "qpi", + "negative_expected_models": [ + "intellectual_archaeology" + ], + "expected_classification": "not_applicable", + "expected_dominant_scarcity": "not_applicable", + "expected_max_depth": "no_call", + "minimum_required_elements": [ + "model_level_hard_exclusion" + ], + "forbidden_elements": [ + "selected IA" + ], + "evaluation_mode": "structured", + "notes": "Round 03.2 hard exclusion trap: only-QPI instruction rejects IA." } ] } diff --git a/tests/regression_index.json b/tests/regression_index.json index 4be3196..c1daee7 100644 --- a/tests/regression_index.json +++ b/tests/regression_index.json @@ -1,33 +1,33 @@ { "index_version": "0.2", - "last_updated": "2026-06-16", + "last_updated": "2026-06-17", "source_of_truth": "per_model_files", "aggregate_file": "tests/regression_cases.json", "models": [ { "model_id": "qpi", "regression_file": "tests/qpi.regression.json", - "case_count": 17, + "case_count": 55, "case_type_counts": { - "positive": 9, - "boundary": 3, - "misuse": 2, - "no_call": 1, - "selector_gate": 1, - "pipeline": 1 + "boundary": 13, + "misuse": 16, + "no_call": 6, + "pipeline": 1, + "positive": 16, + "selector_gate": 3 } }, { "model_id": "intellectual_archaeology", "regression_file": "tests/intellectual_archaeology.regression.json", - "case_count": 17, + "case_count": 19, "case_type_counts": { - "positive": 5, "boundary": 3, "misuse": 3, "no_call": 3, - "selector_gate": 2, - "pipeline": 1 + "pipeline": 1, + "positive": 5, + "selector_gate": 4 } } ]