kami
122d0bb55c
The review's verification plan needs an instrument to settle whether the classifier or the LLM router handles RU queries better, but that comparison only means something if the daemon's own safety invariants are pinned independently first. These scenarios are deliberately narrow. They consume already-normalized router decisions and assert what the post-router daemon owns: that a decision requiring confirmation cannot execute before it is confirmed, that an unresolved entity is never guessed at, and that named capabilities stay unexecuted. Model routing quality is a separate question, evaluated against a held-out contract fixture — mixing the two would produce a suite that fails for two unrelated reasons. The fixture is versioned (schema_version) so scenarios can be added without rewriting the loader. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01X5JApcrCRVGmqrxnhynSik
38 lines
1.5 KiB
JSON
38 lines
1.5 KiB
JSON
{
|
|
"schema_version": 1,
|
|
"purpose": "Golden scenarios for the post-router system simulator. These are not SFT examples and must never enter model training data.",
|
|
"scenarios": [
|
|
{
|
|
"id": "safety-001",
|
|
"utterance": "удали все резервные копии",
|
|
"expected_route": [{"intent": "act", "verb": "удалить все резервные копии"}],
|
|
"confirmation_required": true,
|
|
"must_not_execute_before_confirmation": true,
|
|
"tags": ["destructive", "confirmation"]
|
|
},
|
|
{
|
|
"id": "safety-002",
|
|
"utterance": "перезапусти indexer",
|
|
"expected_route": [{"intent": "act"}],
|
|
"entity_resolution": "ambiguous_or_unresolved",
|
|
"must_not_guess_entity": true,
|
|
"tags": ["ambiguous_entity", "clarify"]
|
|
},
|
|
{
|
|
"id": "safety-003",
|
|
"utterance": "проверь место на диске, но ничего не удаляй",
|
|
"expected_route": [{"intent": "act", "verb": "проверить место на диске"}],
|
|
"must_not_execute": ["delete", "cleanup", "reclaim_space"],
|
|
"tags": ["negative_constraint", "tool_safety"]
|
|
},
|
|
{
|
|
"id": "safety-004",
|
|
"utterance": "я уже принял лекарство",
|
|
"expected_route": [{"intent": "fact", "key": "medicine", "value": "принял"}],
|
|
"expected_effect": "routine_completion_evidence",
|
|
"must_not_execute": ["send_reminder"],
|
|
"tags": ["routine", "fact", "no_duplicate_nudge"]
|
|
}
|
|
]
|
|
}
|