From 787cc56522c7d64a7bcc70e13f1aa5a61dcb5ff8 Mon Sep 17 00:00:00 2001 From: claude Date: Tue, 4 Aug 2026 06:21:40 +0400 Subject: [PATCH] eval: add the held-out ecosystem reach fixture (V-405) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 30 act-shaped Russian utterances, each with the service it must arrive at: 10 hexis, 12 praxis, 8 that must reach neither. The negatives are the half that matters most — without them a router that sent every turn to Hexis would score perfectly. want_capability records which Praxis arm the fn should land on. It is not scored: asserting it would mean asserting an alias table this package cannot import. --- internal/router/eval/ru_ecosystem_v1.json | 284 ++++++++++++++++++++++ 1 file changed, 284 insertions(+) create mode 100644 internal/router/eval/ru_ecosystem_v1.json diff --git a/internal/router/eval/ru_ecosystem_v1.json b/internal/router/eval/ru_ecosystem_v1.json new file mode 100644 index 0000000..848bb68 --- /dev/null +++ b/internal/router/eval/ru_ecosystem_v1.json @@ -0,0 +1,284 @@ +{ + "schema_version": 1, + "name": "ru_ecosystem_v1", + "reference_now": "2026-08-04T10:00:00+03:00", + "notes": [ + "Held out from models/seeds/*.txt, same rule as ru_routing_v1: a case that is verbatim a seed measures memorisation.", + "want_service is where the utterance must ARRIVE, not what it must achieve. Praxis means handlePraxisAct dispatched to a capability; hexis means the text reached resolveEntityReference. Whether the entity exists in Nexus and the capability exists in Hexis is the services' answer, not the router's.", + "The reach model is derived in reach.go from cmd/mavend/actions_act.go. Praxis needs IntentAct AND a fn slot whose value is a praxis capability alias. Hexis needs IntentAct AND non-empty text, and it also fires from hexisBeforeClarify, so a clarified act still reaches it.", + "want_capability is informational and is not scored. It records which praxis arm the fn should land on, so a report can say WHERE a praxis reach went wrong rather than only that it did.", + "none means the utterance must NOT reach either service. Those cases are the expensive direction: a note or a query that lands on a mutating Hexis path is a wrong act, not a missed one." + ], + "cases": [ + { + "id": "eco-ru-001", + "utterance": "выключи свет в спальне", + "lang": "ru", + "want_service": "hexis", + "want_capability": "", + "tags": ["home", "mutating"], + "note": "The canonical act. Verb in the allowlist, entity text after it." + }, + { + "id": "eco-ru-002", + "utterance": "включи свет на кухне, пожалуйста", + "lang": "ru", + "want_service": "hexis", + "want_capability": "", + "tags": ["home", "mutating"], + "note": "" + }, + { + "id": "eco-ru-003", + "utterance": "перезапусти вукунью", + "lang": "ru", + "want_service": "hexis", + "want_capability": "", + "tags": ["homelab", "mutating"], + "note": "Service name in Russian, declined. Nexus owns the alias, the router only has to hand it over." + }, + { + "id": "eco-ru-004", + "utterance": "перезагрузи прокси", + "lang": "ru", + "want_service": "hexis", + "want_capability": "", + "tags": ["homelab", "mutating"], + "note": "" + }, + { + "id": "eco-ru-005", + "utterance": "останови гитею на минуту", + "lang": "ru", + "want_service": "hexis", + "want_capability": "", + "tags": ["homelab", "mutating"], + "note": "Trailing qualifier the entity resolver has to survive." + }, + { + "id": "eco-ru-006", + "utterance": "запусти бэкап на нексусе", + "lang": "ru", + "want_service": "hexis", + "want_capability": "", + "tags": ["homelab", "mutating"], + "note": "" + }, + { + "id": "eco-ru-007", + "utterance": "закрой шторы в комнате", + "lang": "ru", + "want_service": "hexis", + "want_capability": "", + "tags": ["home", "mutating"], + "note": "" + }, + { + "id": "eco-ru-008", + "utterance": "открой гараж", + "lang": "ru", + "want_service": "hexis", + "want_capability": "", + "tags": ["home", "mutating"], + "note": "Two words. The single-token thinner spares it, the verb ending carries the subject." + }, + { + "id": "eco-ru-009", + "utterance": "поставь чайник", + "lang": "ru", + "want_service": "hexis", + "want_capability": "", + "tags": ["home", "mutating"], + "note": "" + }, + { + "id": "eco-ru-010", + "utterance": "выключи телевизор в гостиной", + "lang": "ru", + "want_service": "hexis", + "want_capability": "", + "tags": ["home", "mutating"], + "note": "" + }, + { + "id": "eco-ru-011", + "utterance": "что требует внимания", + "lang": "ru", + "want_service": "praxis", + "want_capability": "list_attention", + "tags": ["attention", "reading"], + "note": "The task's own example. Query-shaped, and the reach model only dispatches to Praxis from IntentAct, so this is the case most likely to miss." + }, + { + "id": "eco-ru-012", + "utterance": "что сейчас требует внимания", + "lang": "ru", + "want_service": "praxis", + "want_capability": "list_attention", + "tags": ["attention", "reading"], + "note": "" + }, + { + "id": "eco-ru-013", + "utterance": "что нового по проектам", + "lang": "ru", + "want_service": "praxis", + "want_capability": "list_attention", + "tags": ["attention", "reading"], + "note": "" + }, + { + "id": "eco-ru-014", + "utterance": "что изменилось за сегодня", + "lang": "ru", + "want_service": "praxis", + "want_capability": "list_changes", + "tags": ["attention", "reading"], + "note": "" + }, + { + "id": "eco-ru-015", + "utterance": "покажи изменения", + "lang": "ru", + "want_service": "praxis", + "want_capability": "list_changes", + "tags": ["attention", "reading"], + "note": "" + }, + { + "id": "eco-ru-016", + "utterance": "отметь это как сделанное", + "lang": "ru", + "want_service": "praxis", + "want_capability": "resolve_item", + "tags": ["lifecycle", "mutating"], + "note": "The task's second example. The item id is the value slot and is missing here, so the capability asks for it — reaching Praxis with no id is still reaching Praxis." + }, + { + "id": "eco-ru-017", + "utterance": "готово, закрывай", + "lang": "ru", + "want_service": "praxis", + "want_capability": "resolve_item", + "tags": ["lifecycle", "mutating"], + "note": "" + }, + { + "id": "eco-ru-018", + "utterance": "принято, я это видел", + "lang": "ru", + "want_service": "praxis", + "want_capability": "acknowledge_item", + "tags": ["lifecycle", "mutating"], + "note": "Acknowledged is not resolved. The alias must not collapse into the resolve arm." + }, + { + "id": "eco-ru-019", + "utterance": "игнорировать это пока", + "lang": "ru", + "want_service": "praxis", + "want_capability": "ignore_item", + "tags": ["lifecycle", "mutating"], + "note": "" + }, + { + "id": "eco-ru-020", + "utterance": "закрепи этот пункт", + "lang": "ru", + "want_service": "praxis", + "want_capability": "pin_item", + "tags": ["lifecycle", "mutating"], + "note": "" + }, + { + "id": "eco-ru-021", + "utterance": "что там с нексусом", + "lang": "ru", + "want_service": "praxis", + "want_capability": "entity_attention", + "tags": ["attention", "reading"], + "note": "Scoped attention. The alias list for this arm is grammar names only, so free Russian reaching it depends on a grammar that does not exist yet." + }, + { + "id": "eco-ru-022", + "utterance": "как дела у праксиса", + "lang": "ru", + "want_service": "praxis", + "want_capability": "entity_attention", + "tags": ["attention", "reading"], + "note": "" + }, + { + "id": "eco-ru-023", + "utterance": "напомни завтра в девять позвонить маме", + "lang": "ru", + "want_service": "none", + "want_capability": "", + "tags": ["negative"], + "note": "A reminder is Maven's own store. Nothing outside her hears it." + }, + { + "id": "eco-ru-024", + "utterance": "запиши что мне понравился этот подход", + "lang": "ru", + "want_service": "none", + "want_capability": "", + "tags": ["negative"], + "note": "A note carrying a verb-shaped word. It must not become an act." + }, + { + "id": "eco-ru-025", + "utterance": "какая завтра погода", + "lang": "ru", + "want_service": "none", + "want_capability": "", + "tags": ["negative"], + "note": "" + }, + { + "id": "eco-ru-026", + "utterance": "мой рост сто восемьдесят два", + "lang": "ru", + "want_service": "none", + "want_capability": "", + "tags": ["negative"], + "note": "A fact about him. The personal boundary means this never leaves the box." + }, + { + "id": "eco-ru-027", + "utterance": "как ты сегодня", + "lang": "ru", + "want_service": "none", + "want_capability": "", + "tags": ["negative"], + "note": "" + }, + { + "id": "eco-ru-028", + "utterance": "выключи", + "lang": "ru", + "want_service": "none", + "want_capability": "", + "tags": ["negative", "ambiguous"], + "note": "A verb with no target. Reaching Hexis with an empty entity text is the failure this case guards: it must clarify, not resolve." + }, + { + "id": "eco-ru-029", + "utterance": "сделай это", + "lang": "ru", + "want_service": "none", + "want_capability": "", + "tags": ["negative", "ambiguous"], + "note": "The known dangerous case. It carries text, so under the current gates it does reach Hexis — the fixture says it should not, and that disagreement is the finding." + }, + { + "id": "eco-ru-030", + "utterance": "во сколько у меня встреча", + "lang": "ru", + "want_service": "none", + "want_capability": "", + "tags": ["negative"], + "note": "Stage 0 routes agenda questions to query. The calendar is Maven's, not Praxis's." + } + ] +}