diff --git a/internal/router/eval/eval.go b/internal/router/eval/eval.go index f4bcb78..59dc0cb 100644 --- a/internal/router/eval/eval.go +++ b/internal/router/eval/eval.go @@ -286,6 +286,12 @@ func Score(ctx context.Context, name string, r Router, f Fixture) (Report, error if c.WantSource != nil && err == nil { rep.SourceTotal++ switch { + case !o.IntentOK: + // The route never got to a destination, so a match on the + // SourceUnknown floor here would be a coincidence scored as a + // win: a clarify names nothing and would satisfy "" for free. + o.SourceReason = fmt.Sprintf("no destination, route missed %q", c.Intent) + rep.SourceConfusion[string(*c.WantSource)+"→(no route)"]++ case d.Source == *c.WantSource: rep.SourceHit++ default: diff --git a/internal/router/eval/eval_test.go b/internal/router/eval/eval_test.go index 8b8ee8b..879388c 100644 --- a/internal/router/eval/eval_test.go +++ b/internal/router/eval/eval_test.go @@ -266,6 +266,11 @@ func baselineGrammars(acts router.ActMatcher) []router.Grammar { // Same order as buildRouter (voicewire.go). The fixture is only worth // anything while its grammar set is the daemon's grammar set. grammars = append(grammars, router.AgendaQueryGrammars()...) + // After the agenda rules and before the feed and list rules, same as + // voicewire.go: "что такое лента" is a definition question and the feed + // rule would claim it on the noun alone (V-655). Missing here until V-659, + // so the fixture was scoring a grammar set the daemon does not run. + grammars = append(grammars, router.WorldQueryGrammars()...) grammars = append(grammars, router.FeedQueryGrammar()) // The list side of the same exposure: a phrasing with no possessive in it // ("список дел") routed system and never reached queryTasks (Vikunja #467). diff --git a/internal/router/eval/ru_routing_v1.json b/internal/router/eval/ru_routing_v1.json index 18de608..d2d4b5f 100644 --- a/internal/router/eval/ru_routing_v1.json +++ b/internal/router/eval/ru_routing_v1.json @@ -6,37 +6,41 @@ "Held-out routing contract. Every utterance here is absent from models/seeds/*.txt (TestFixtureIsHeldOut enforces it verbatim) — scoring a classifier on its own seed phrases measures memorisation, not routing.", "This is a CONTRACT, not a snapshot of current behaviour. Cases the classifier cascade fails today are expected to stay in the file and fail loudly; that failure count is the number Vikunja #319 compares against the LLM router before #320 flips the default.", "Slot expectations are deployment-independent on purpose. want_fn is a boolean (the act must resolve to SOME allowlisted fn) because the allowlist lives in deploy config, not here. want_fact_key names the loop's rule keys (water/meal/sleep/break/shower) — a fact that lands under the wrong key silently starves the predicate that reads it.", - "want_clarify cases carry intent \"\": the contract is that the router refuses rather than guesses. A confident answer there is a worse failure than a miss." + "want_clarify cases carry intent \"\": the contract is that the router refuses rather than guesses. A confident answer there is a worse failure than a miss.", + "want_source is the second half of a route (V-655). It is present only on query cases, because no other intent reaches queryWalk, and absent there means absent rather than SourceUnknown. Empty is a label and not a gap: it asserts that the decider must name nothing and let the daemon walk the whole chain in order, his data first.", + "Seven cases assert that floor and six of them are homelab operations. They cluster because SourceRecall, SourceNetwork and SourceAttention overlap on every question about the box: mavpoll writes its observations into the fact store recall reads. That is a finding about the enum, not a gap in the labelling.", + "ru-query-020 and ru-query-024 are the same utterance, as are ru-query-021 and ru-query-025. Both pairs differ in tags and note only, so both pairs are counted twice in every number this fixture reports.", + "Every want_source is the destination that SHOULD claim the turn, which on ru-query-026 through 030 is not the one that did. Those five were observed failing on the box on 2026-08-07 (docs/evals/2026-08-07-week-of-usage.md). A fixture that passes on the day it is written measures nothing." ], "cases": [ - { "id": "ru-query-001", "utterance": "сколько воды я выпил с утра", "lang": "ru", "intent": "query", "tags": ["aggregate"] }, - { "id": "ru-query-002", "utterance": "я сегодня вообще пил воду", "lang": "ru", "intent": "query", "tags": ["hard", "fact-shaped"], "note": "past-tense fact lexicon in a question — the classifier's fact centroid pulls this hard" }, - { "id": "ru-query-003", "utterance": "во сколько я лёг вчера", "lang": "ru", "intent": "query", "tags": ["temporal"] }, - { "id": "ru-query-004", "utterance": "давно я не тренировался", "lang": "ru", "intent": "query", "tags": ["hard", "no-question-word"] }, - { "id": "ru-query-005", "utterance": "напоминания на завтра есть", "lang": "ru", "intent": "query", "tags": ["hard", "reminder-shaped"], "note": "asks about reminders, does not create one" }, - { "id": "ru-query-006", "utterance": "что я записывал про кота", "lang": "ru", "intent": "query", "tags": ["recall"] }, - { "id": "ru-query-007", "utterance": "сколько раз я ел вчера", "lang": "ru", "intent": "query", "tags": ["aggregate", "hard"] }, - { "id": "ru-query-008", "utterance": "мой вес за последний месяц", "lang": "ru", "intent": "query", "tags": ["no-verb"] }, - { "id": "ru-query-009", "utterance": "когда я в последний раз принимал витамины", "lang": "ru", "intent": "query", "tags": ["temporal"] }, - { "id": "ru-query-010", "utterance": "есть новости по бэкапу базы", "lang": "ru", "intent": "query", "tags": ["homelab"] }, - { "id": "ru-query-011", "utterance": "почему сервер тормозит", "lang": "ru", "intent": "query", "tags": ["homelab", "hard"], "note": "diagnostic question, not a chat opener" }, - { "id": "ru-query-012", "utterance": "какие заметки я оставил про полив", "lang": "ru", "intent": "query", "tags": ["recall"] }, - { "id": "ru-query-013", "utterance": "во сколько у меня встреча", "lang": "ru", "intent": "query", "tags": ["calendar"] }, - { "id": "ru-query-019", "utterance": "что у меня стоит в календаре на послезавтра", "lang": "ru", "intent": "query", "tags": ["calendar", "hard"], "note": "agenda, not the clock: the daemon answers this from CalendarEvents inside the query branch, so the clock/date system rule must not swallow it" }, - { "id": "ru-query-022", "utterance": "какие планы на завтра?", "lang": "ru", "intent": "query", "tags": ["calendar"], "note": "the same agenda question as ru-query-019 aimed at another day; it answered \u043f\u043e\u043a\u0430 \u043d\u0435 \u0443\u043c\u0435\u044e on the deployed daemon while the today form worked (Vikunja #471)" }, - { "id": "ru-query-023", "utterance": "\u043a\u043e\u0433\u0434\u0430 \u043f\u043b\u0430\u043d\u0451\u0440\u043a\u0430?", "lang": "ru", "intent": "query", "tags": ["calendar", "hard"], "note": "a named event with no calendar word — the noun is the only signal that this is a question about his day" }, - { "id": "ru-query-024", "utterance": "что дальше?", "lang": "ru", "intent": "query", "tags": ["calendar", "no-question-word"], "note": "the rest of the day, with no possessive and no plan word to anchor on; the model called it a fact and the write had to be caught downstream (Vikunja #498)" }, - { "id": "ru-query-025", "utterance": "расскажи про битву при Ватерлоо", "lang": "ru", "intent": "query", "tags": ["world", "no-question-word"], "note": "a narrative request carries no question mark and no interrogative, so it routed fact; contrast ru-chat-003, where the same verb asks for a joke" }, - { "id": "ru-query-014", "utterance": "я успеваю до дедлайна", "lang": "ru", "intent": "query", "tags": ["hard", "no-question-word"] }, - { "id": "ru-query-015", "utterance": "сколько я прошёл шагов", "lang": "ru", "intent": "query", "tags": ["aggregate"] }, - { "id": "ru-query-016", "utterance": "покажи давление за неделю", "lang": "ru", "intent": "query", "tags": ["hard", "imperative"], "note": "imperative form but a read — must not route to act" }, - { "id": "ru-query-017", "utterance": "чем я занимался в среду", "lang": "ru", "intent": "query", "tags": ["hard", "chat-shaped"] }, - { "id": "ru-query-018", "utterance": "хватает ли места под новые бэкапы", "lang": "ru", "intent": "query", "tags": ["homelab"] }, - { "id": "ru-query-020", "utterance": "что дальше?", "lang": "ru", "intent": "query", "tags": ["agenda", "hard"], "note": "the rest of the day, with no interrogative the model can read as a question — it routed fact until a stage 0 rule claimed it (V-498)" }, - { "id": "ru-query-021", "utterance": "расскажи про битву при Ватерлоо", "lang": "ru", "intent": "query", "tags": ["world", "hard"], "note": "a world question phrased as an instruction. It routed fact, and the fact gate had to catch the write (V-498)" }, - { "id": "en-query-001", "utterance": "did I take my vitamins today", "lang": "en", "intent": "query", "tags": ["fact-shaped"] }, - { "id": "en-query-002", "utterance": "how long since the last backup finished", "lang": "en", "intent": "query", "tags": ["temporal"] }, - { "id": "en-query-003", "utterance": "show me this week's weight", "lang": "en", "intent": "query", "tags": ["imperative"] }, + { "id": "ru-query-001", "utterance": "сколько воды я выпил с утра", "lang": "ru", "intent": "query", "want_source": "recall", "tags": ["aggregate"] }, + { "id": "ru-query-002", "utterance": "я сегодня вообще пил воду", "lang": "ru", "intent": "query", "want_source": "recall", "tags": ["hard", "fact-shaped"], "note": "past-tense fact lexicon in a question — the classifier's fact centroid pulls this hard" }, + { "id": "ru-query-003", "utterance": "во сколько я лёг вчера", "lang": "ru", "intent": "query", "want_source": "recall", "tags": ["temporal"] }, + { "id": "ru-query-004", "utterance": "давно я не тренировался", "lang": "ru", "intent": "query", "want_source": "recall", "tags": ["hard", "no-question-word"] }, + { "id": "ru-query-005", "utterance": "напоминания на завтра есть", "lang": "ru", "intent": "query", "want_source": "", "tags": ["hard", "reminder-shaped"], "note": "asks about reminders, does not create one. want_source is the floor on purpose: no query source reads the reminder store, and day-plan is SourceCalendar over a table this box does not write." }, + { "id": "ru-query-006", "utterance": "что я записывал про кота", "lang": "ru", "intent": "query", "want_source": "recall", "tags": ["recall"] }, + { "id": "ru-query-007", "utterance": "сколько раз я ел вчера", "lang": "ru", "intent": "query", "want_source": "recall", "tags": ["aggregate", "hard"] }, + { "id": "ru-query-008", "utterance": "мой вес за последний месяц", "lang": "ru", "intent": "query", "want_source": "recall", "tags": ["no-verb"] }, + { "id": "ru-query-009", "utterance": "когда я в последний раз принимал витамины", "lang": "ru", "intent": "query", "want_source": "recall", "tags": ["temporal"] }, + { "id": "ru-query-010", "utterance": "есть новости по бэкапу базы", "lang": "ru", "intent": "query", "want_source": "", "tags": ["homelab"], "note": "recall, attention and network can each answer it, because mavpoll writes its netdata and uptime-kuma observations into the fact store recall reads. Naming one takes the other two off the turn." }, + { "id": "ru-query-011", "utterance": "почему сервер тормозит", "lang": "ru", "intent": "query", "want_source": "", "tags": ["homelab", "hard"], "note": "diagnostic question, not a chat opener. network holds the box and attention holds the alarm about the box. The utterance does not choose, so neither does the label." }, + { "id": "ru-query-012", "utterance": "какие заметки я оставил про полив", "lang": "ru", "intent": "query", "want_source": "recall", "tags": ["recall"] }, + { "id": "ru-query-013", "utterance": "во сколько у меня встреча", "lang": "ru", "intent": "query", "want_source": "calendar", "tags": ["calendar"] }, + { "id": "ru-query-019", "utterance": "что у меня стоит в календаре на послезавтра", "lang": "ru", "intent": "query", "want_source": "calendar", "tags": ["calendar", "hard"], "note": "agenda, not the clock: the daemon answers this from CalendarEvents inside the query branch, so the clock/date system rule must not swallow it" }, + { "id": "ru-query-022", "utterance": "какие планы на завтра?", "lang": "ru", "intent": "query", "want_source": "calendar", "tags": ["calendar"], "note": "the same agenda question as ru-query-019 aimed at another day; it answered \u043f\u043e\u043a\u0430 \u043d\u0435 \u0443\u043c\u0435\u044e on the deployed daemon while the today form worked (Vikunja #471)" }, + { "id": "ru-query-023", "utterance": "\u043a\u043e\u0433\u0434\u0430 \u043f\u043b\u0430\u043d\u0451\u0440\u043a\u0430?", "lang": "ru", "intent": "query", "want_source": "calendar", "tags": ["calendar", "hard"], "note": "a named event with no calendar word — the noun is the only signal that this is a question about his day" }, + { "id": "ru-query-024", "utterance": "что дальше?", "lang": "ru", "intent": "query", "want_source": "calendar", "tags": ["calendar", "no-question-word"], "note": "the rest of the day, with no possessive and no plan word to anchor on; the model called it a fact and the write had to be caught downstream (Vikunja #498)" }, + { "id": "ru-query-025", "utterance": "расскажи про битву при Ватерлоо", "lang": "ru", "intent": "query", "want_source": "world", "tags": ["world", "no-question-word"], "note": "a narrative request carries no question mark and no interrogative, so it routed fact; contrast ru-chat-003, where the same verb asks for a joke" }, + { "id": "ru-query-014", "utterance": "я успеваю до дедлайна", "lang": "ru", "intent": "query", "want_source": "", "tags": ["hard", "no-question-word"], "note": "a deadline lives in the task list, the calendar or Praxis depending on where he put it. The destination depends on his data, not on his words." }, + { "id": "ru-query-015", "utterance": "сколько я прошёл шагов", "lang": "ru", "intent": "query", "want_source": "recall", "tags": ["aggregate"] }, + { "id": "ru-query-016", "utterance": "покажи давление за неделю", "lang": "ru", "intent": "query", "want_source": "recall", "tags": ["hard", "imperative"], "note": "imperative form but a read — must not route to act" }, + { "id": "ru-query-017", "utterance": "чем я занимался в среду", "lang": "ru", "intent": "query", "want_source": "recall", "tags": ["hard", "chat-shaped"] }, + { "id": "ru-query-018", "utterance": "хватает ли места под новые бэкапы", "lang": "ru", "intent": "query", "want_source": "", "tags": ["homelab"], "note": "disk headroom. network is the only source that reads the box, but the phrasing is a capacity question and not a LAN one." }, + { "id": "ru-query-020", "utterance": "что дальше?", "lang": "ru", "intent": "query", "want_source": "calendar", "tags": ["agenda", "hard"], "note": "the rest of the day, with no interrogative the model can read as a question — it routed fact until a stage 0 rule claimed it (V-498)" }, + { "id": "ru-query-021", "utterance": "расскажи про битву при Ватерлоо", "lang": "ru", "intent": "query", "want_source": "world", "tags": ["world", "hard"], "note": "a world question phrased as an instruction. It routed fact, and the fact gate had to catch the write (V-498)" }, + { "id": "en-query-001", "utterance": "did I take my vitamins today", "lang": "en", "intent": "query", "want_source": "recall", "tags": ["fact-shaped"] }, + { "id": "en-query-002", "utterance": "how long since the last backup finished", "lang": "en", "intent": "query", "want_source": "", "tags": ["temporal"], "note": "the completion time is a fact the poller wrote, so recall answers it. A person asking this wants the operational answer. Both are true." }, + { "id": "en-query-003", "utterance": "show me this week's weight", "lang": "en", "intent": "query", "want_source": "recall", "tags": ["imperative"] }, { "id": "ru-fact-001", "utterance": "только что выпил кружку воды", "lang": "ru", "intent": "fact", "want_fact_key": "water" }, { "id": "ru-fact-002", "utterance": "воды попил наконец", "lang": "ru", "intent": "fact", "want_fact_key": "water", "tags": ["inverted"] }, @@ -106,6 +110,11 @@ { "id": "amb-005", "utterance": "потом", "lang": "ru", "want_clarify": true, "tags": ["ambiguous", "filler"] }, { "id": "amb-006", "utterance": "the thing from earlier", "lang": "en", "want_clarify": true, "tags": ["ambiguous", "anaphora"] }, { "id": "amb-007", "utterance": "напомни", "lang": "ru", "want_clarify": true, "tags": ["ambiguous", "reminder"], "note": "the reminder verb and nothing else — she knows the shape of the request and not one thing about it. Answered 'не получилось разобрать время напоминания' on the box until V-548: the subjectless-reminder gate tested Slots.Text == \"\", and fillSlots had put the verb in that slot" }, - { "id": "amb-008", "utterance": "ну напомни же", "lang": "ru", "want_clarify": true, "tags": ["ambiguous", "reminder", "filler"], "note": "the same request wrapped in particles, which is why filler_particles is a lexicon set — without it the particles read as the subject" } + { "id": "amb-008", "utterance": "ну напомни же", "lang": "ru", "want_clarify": true, "tags": ["ambiguous", "reminder", "filler"], "note": "the same request wrapped in particles, which is why filler_particles is a lexicon set — without it the particles read as the subject" }, + { "id": "ru-query-026", "utterance": "что такое TCP?", "lang": "ru", "intent": "query", "want_source": "world", "tags": ["world", "regression"], "note": "weather claimed it on 2026-08-07 and answered \"для какого города?\", because it read one percent closer than the leftover seeds. WorldQueryGrammars claims it at stage 0 now." }, + { "id": "ru-query-027", "utterance": "сколько будет 17 на 23?", "lang": "ru", "intent": "query", "want_source": "world", "tags": ["world", "arithmetic", "regression"], "note": "same source, same day, same answer about a city. Arithmetic is not a place." }, + { "id": "ru-query-028", "utterance": "какой у меня любимый язык?", "lang": "ru", "intent": "query", "want_source": "recall", "tags": ["recall", "possessive", "regression"], "note": "the feed answered it with kernel headlines. \"у меня\" is the whole signal and it points inward." }, + { "id": "ru-query-029", "utterance": "кто такой Линус Торвальдс?", "lang": "ru", "intent": "query", "want_source": "world", "tags": ["world", "person", "regression"], "note": "the personal boundary answered \"не нашла у тебя такой записи\". A named public person is not his data." }, + { "id": "ru-query-030", "utterance": "что там с бэкапами?", "lang": "ru", "intent": "query", "want_source": "", "tags": ["homelab", "regression"], "note": "search claimed it, which inverts the boundary outward. The fix is the chain order and not a destination: recall, attention and network all answer it, same as ru-query-010." } ] }