diff --git a/CLAUDE.md b/CLAUDE.md index 752fa1f..1a0dbda 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -214,6 +214,20 @@ New fixture cases ru-query-024 and ru-query-025. Classifier + ONNX baseline **56 58/82 (70.7%)**, no case regressed, no new false clarify. The LLM arm was not measured (no llama-server in that run), so judge it again before quoting a cascade number. +Praxis taken off the model, 05-08-2026 (V-516). `PraxisGrammars()` +(`internal/router/praxis.go`, wired in `buildRouter` before the capture marker because +"отметь" is a capture verb) fills `Slots.Fn` with a Praxis capability name. Praxis reach +was **0/12 and structurally so**: `handlePraxisAct` compares `Slots.Fn` to a capability +alias, and that slot is filled from the deployment's enabled tool names, which no Praxis +alias is on. Measured **16/30 → 27/30 overall, praxis 0/12 → 11/12, lifecycle 0/5 → 5/5** +(`docs/evals/2026-08-05-praxis-reach.md`). Two rules to know before editing: a **stative** +lifecycle word ("готово", "принято") needs an item named beside it, while a bare +**imperative** ("закрывай") may ask which one. The bare arm additionally requires that +the sentence name no object of its own, or "закрой шторы в комнате" goes to Praxis instead +of the house. A demonstrative ("отметь это как сделанное") resolves against +`h.surfacedItems` only when exactly one item was spoken. Otherwise the turn goes back to +the cascade rather than transitioning the wrong item. + ## LLM output contract All phrasing paths emit `{"response":"...","mood":"..."}` (parsed in `replier_llm.go` and diff --git a/cmd/mavend/ecosystem_acts.go b/cmd/mavend/ecosystem_acts.go index 0277479..1f5438a 100644 --- a/cmd/mavend/ecosystem_acts.go +++ b/cmd/mavend/ecosystem_acts.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "log" + "strconv" "strings" "time" @@ -105,6 +106,13 @@ func (h *reactiveHandler) handlePraxisAct(ctx context.Context, dec router.Decisi ctx = withCorrelationID(ctx, newCorrelationID()) } px := h.ecosystem.praxis + dec, ok := h.resolveSurfacedPosition(dec) + if !ok { + // A demonstrative with no digest behind it. "я это сделал" is a sentence + // about his day, so the rest of the cascade gets it back rather than + // hearing "какой пункт?" for something that was never about a пункт. + return "" + } for _, capability := range praxisCapabilities { for _, alias := range capability.aliases() { if alias == dec.Slots.Fn { @@ -170,6 +178,7 @@ func (listAttentionCapability) handle(ctx context.Context, h *reactiveHandler, p } h.recordPraxisTrace(ctx, "list_attention", started, map[string]any{"count": len(items)}) var parts []string + var spoken []string for _, item := range items { title, _ := item["title"].(string) // importance arrives as JSON number ⇒ float64 over the HTTP contract. @@ -194,11 +203,15 @@ func (listAttentionCapability) handle(ctx context.Context, h *reactiveHandler, p // (ECOSYSTEM-SPEC.md §2.3: surfaced != acknowledged). Best-effort: // a failed surface call must not block delivering the digest. if id, ok := item["id"].(string); ok && id != "" { + // Recorded in the order she says them, and only for items she could + // say: an item skipped above has no position in what he heard (#516). + spoken = append(spoken, id) if _, err := px.Surface(ctx, id); err != nil { log.Printf("ecosystem: praxis surface %s: %v", id, err) } } } + h.rememberSurfaced(spoken) if len(parts) == 0 { // Praxis returned items and not one of them could be said. "ничего не // требует внимания" is the honest answer; the list line would render as @@ -835,3 +848,62 @@ func (h *reactiveHandler) attentionCannotTell(ctx context.Context, px *praxisCli } return "" } + +// rememberSurfaced records the item ids she just read out, replacing whatever the +// previous digest left. Called with the ids in speaking order (Vikunja #516). +func (h *reactiveHandler) rememberSurfaced(ids []string) { + h.mu.Lock() + defer h.mu.Unlock() + h.surfacedItems = ids +} + +// resolveSurfacedPosition turns a positional item reference into a Praxis item +// id, using the list she last read out. +// +// The router names a position and not an id, because only the daemon has the +// list: PraxisGrammars fills the value slot with "2", "last" or "this". An id is +// left alone, since "item_ab12" is already one. +// +// The second return says whether the turn is still Praxis's. A position that +// names nothing keeps the turn and clears the slot, so the capability answers its +// own "какой пункт?" — he said "второй пункт" and deserves to hear that there is +// no second one. A demonstrative that resolves to nothing gives the turn BACK, +// because "я это сделал" was probably never about a пункт at all. "это" also +// needs the list to hold exactly one item: pointing at one of five is a guess, +// and a wrong guess here transitions the wrong item. +func (h *reactiveHandler) resolveSurfacedPosition(dec router.Decision) (router.Decision, bool) { + ref := dec.Slots.Value + if ref == "" || strings.HasPrefix(ref, "item") { + return dec, true + } + h.mu.Lock() + ids := h.surfacedItems + h.mu.Unlock() + + idx := -1 + switch { + case ref == "this": + if len(ids) != 1 { + log.Printf("ecosystem: praxis \"это\" has no single item (%d surfaced)", len(ids)) + return dec, false + } + idx = 0 + case ref == "last": + idx = len(ids) - 1 + default: + n, err := strconv.Atoi(ref) + if err != nil || n < 1 { + // Neither a position nor an id: leave it for the capability to + // reject rather than silently rewriting what he said. + return dec, true + } + idx = n - 1 + } + if idx < 0 || idx >= len(ids) { + log.Printf("ecosystem: praxis position %q has no item (%d surfaced)", ref, len(ids)) + dec.Slots.Value = "" + return dec, true + } + dec.Slots.Value = ids[idx] + return dec, true +} diff --git a/cmd/mavend/fakeecosystem_test.go b/cmd/mavend/fakeecosystem_test.go index cb34528..3f6c401 100644 --- a/cmd/mavend/fakeecosystem_test.go +++ b/cmd/mavend/fakeecosystem_test.go @@ -178,6 +178,14 @@ func (fs *fakeServer) Requests() []capturedRequest { return out } +// ResetRequests drops the captured requests, so a test can assert about one +// turn without subtracting the setup turn's calls. +func (fs *fakeServer) ResetRequests() { + fs.mu.Lock() + defer fs.mu.Unlock() + fs.requests = nil +} + func jsonHandler(status int, body string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/cmd/mavend/praxis_position_test.go b/cmd/mavend/praxis_position_test.go new file mode 100644 index 0000000..8a2fb37 --- /dev/null +++ b/cmd/mavend/praxis_position_test.go @@ -0,0 +1,160 @@ +package main + +import ( + "context" + "strings" + "testing" +) + +// "отметь второй пункт" names a position, and only the daemon knows which item +// that is. The router fills the value slot with "2"; this is where it becomes an +// item id (Vikunja #516). +func TestPositionResolvesAgainstTheLastSpokenList(t *testing.T) { + praxis := newFakePraxis(t, `[ + {"id":"item_a","title":"диск заканчивается"}, + {"id":"item_b","title":"бэкап не прошёл"}, + {"id":"item_c","title":"сертификат истекает"} + ]`) + h := newPraxisTestHandler(t, praxis) + + if reply := h.handlePraxisAct(context.Background(), praxisActDec("list_attention")); reply == "" { + t.Fatal("attention returned nothing") + } + + cases := []struct{ ref, wantItem string }{ + {"2", "item_b"}, + {"1", "item_a"}, + {"last", "item_c"}, + } + for _, c := range cases { + praxis.ResetRequests() + reply := h.handlePraxisAct(context.Background(), praxisItemDec("acknowledge_item", c.ref)) + if !strings.Contains(reply, "принято") { + t.Errorf("ref %q: reply %q", c.ref, reply) + } + if !requestedPathContaining(praxis, c.wantItem) { + t.Errorf("ref %q did not acknowledge %s; paths %v", c.ref, c.wantItem, paths(praxis)) + } + } +} + +// A position past the end must not acknowledge the wrong item. It asks. +func TestPositionPastTheEndAsksInsteadOfGuessing(t *testing.T) { + praxis := newFakePraxis(t, `[{"id":"item_a","title":"диск заканчивается"}]`) + h := newPraxisTestHandler(t, praxis) + h.handlePraxisAct(context.Background(), praxisActDec("list_attention")) + + praxis.ResetRequests() + reply := h.handlePraxisAct(context.Background(), praxisItemDec("resolve_item", "4")) + if !strings.Contains(reply, "какой пункт") { + t.Errorf("a position with no item should ask, got %q", reply) + } + if requestedPathContaining(praxis, "item_a") { + t.Error("the only surfaced item was resolved for a position that did not name it") + } +} + +// No digest yet means no positions. Nothing is mutated. +func TestPositionWithNoSpokenListAsks(t *testing.T) { + praxis := newFakePraxis(t, `[]`) + h := newPraxisTestHandler(t, praxis) + + reply := h.handlePraxisAct(context.Background(), praxisItemDec("acknowledge_item", "1")) + if !strings.Contains(reply, "какой пункт") { + t.Errorf("want the ask, got %q", reply) + } +} + +// An explicit id is not a position and passes through untouched. +func TestExplicitItemIDIsNotRewritten(t *testing.T) { + praxis := newFakePraxis(t, `[{"id":"item_a","title":"диск"}]`) + h := newPraxisTestHandler(t, praxis) + h.handlePraxisAct(context.Background(), praxisActDec("list_attention")) + + praxis.ResetRequests() + h.handlePraxisAct(context.Background(), praxisItemDec("pin_item", "item_zz")) + if !requestedPathContaining(praxis, "item_zz") { + t.Errorf("the id he gave was not the one called; paths %v", paths(praxis)) + } +} + +// An item Praxis sent without a title is never spoken, so it holds no position. +func TestUnspokenItemsHoldNoPosition(t *testing.T) { + praxis := newFakePraxis(t, `[ + {"id":"item_silent"}, + {"id":"item_said","title":"бэкап не прошёл"} + ]`) + h := newPraxisTestHandler(t, praxis) + h.handlePraxisAct(context.Background(), praxisActDec("list_attention")) + + praxis.ResetRequests() + h.handlePraxisAct(context.Background(), praxisItemDec("acknowledge_item", "1")) + if !requestedPathContaining(praxis, "item_said") { + t.Errorf("position 1 is the first item she SAID; paths %v", paths(praxis)) + } +} + +// The item id travels in the POST body, so that is what these read. +func paths(f *fakeServer) []string { + var out []string + for _, r := range f.Requests() { + out = append(out, r.Path+" "+string(r.Body)) + } + return out +} + +func requestedPathContaining(f *fakeServer, want string) bool { + for _, r := range f.Requests() { + if strings.Contains(string(r.Body), want) { + return true + } + } + return false +} + +// "отметь это как сделанное" after a one-item digest points at that item. +func TestDemonstrativeResolvesWhenOneItemWasSpoken(t *testing.T) { + praxis := newFakePraxis(t, `[{"id":"item_only","title":"бэкап не прошёл"}]`) + h := newPraxisTestHandler(t, praxis) + h.handlePraxisAct(context.Background(), praxisActDec("list_attention")) + + praxis.ResetRequests() + reply := h.handlePraxisAct(context.Background(), praxisItemDec("acknowledge_item", "this")) + if !strings.Contains(reply, "принято") { + t.Errorf("reply %q", reply) + } + if !requestedPathContaining(praxis, "item_only") { + t.Errorf("the one surfaced item was not acknowledged; paths %v", paths(praxis)) + } +} + +// Pointing at one of several is a guess, and a wrong guess transitions the wrong +// item. The turn goes back to the cascade instead. +func TestDemonstrativeWithSeveralItemsGivesTheTurnBack(t *testing.T) { + praxis := newFakePraxis(t, `[ + {"id":"item_a","title":"диск"}, + {"id":"item_b","title":"бэкап"} + ]`) + h := newPraxisTestHandler(t, praxis) + h.handlePraxisAct(context.Background(), praxisActDec("list_attention")) + + praxis.ResetRequests() + if reply := h.handlePraxisAct(context.Background(), praxisItemDec("resolve_item", "this")); reply != "" { + t.Errorf("want a fall-through, got %q", reply) + } + for _, p := range paths(praxis) { + if strings.Contains(p, "resolve") { + t.Error("an ambiguous demonstrative resolved an item anyway") + } + } +} + +// "я это сделал" with no digest behind it is a sentence about his day. +func TestDemonstrativeWithNoDigestGivesTheTurnBack(t *testing.T) { + praxis := newFakePraxis(t, `[]`) + h := newPraxisTestHandler(t, praxis) + + if reply := h.handlePraxisAct(context.Background(), praxisItemDec("resolve_item", "this")); reply != "" { + t.Errorf("want a fall-through, got %q", reply) + } +} diff --git a/cmd/mavend/voice.go b/cmd/mavend/voice.go index 9b914a2..7cf1410 100644 --- a/cmd/mavend/voice.go +++ b/cmd/mavend/voice.go @@ -158,6 +158,15 @@ type reactiveHandler struct { pendingRoutine *pendingRoutineConfirm // routine proposal awaiting y/n pendingHexis *pendingHexisExec // mutating Hexis capability awaiting y/n + // surfacedItems — the Praxis item ids she last read out, in the order she + // read them, so "отметь второй пункт" has a second pункт to mean (Vikunja + // #516). Same single-slot posture as pending above: the next attention digest + // replaces the list, because a position only refers to the last one spoken. + // No TTL — a stale position resolves to an item that Praxis will report as + // already acknowledged, which is a harmless answer, unlike a stale + // confirmation that would execute something. + surfacedItems []string + ecosystem *ecosystemWiring // nexus + hexis + praxis clients } diff --git a/cmd/mavend/voicewire.go b/cmd/mavend/voicewire.go index fbbd7c4..75ccc36 100644 --- a/cmd/mavend/voicewire.go +++ b/cmd/mavend/voicewire.go @@ -392,6 +392,10 @@ func buildRouter(emb router.Embedder, acts router.ActMatcher, threshold float64, grammars = append(grammars, router.TaskListGrammar()) grammars = append(grammars, router.ListGrammars()...) grammars = append(grammars, router.ReminderGrammar()) + // Before the capture marker, because "отметь" is a capture verb and "отметь + // второй пункт" is not a note. The Praxis rules are the narrower claim — a + // lifecycle verb AND an item named — so they get first refusal (Vikunja #516). + grammars = append(grammars, router.PraxisGrammars()...) // Last, and it matches any utterance shape — its Build is the filter. An // explicit capture marker beats the model, which called it an act and // rewrote the task text (Vikunja #467). After the rules above because a diff --git a/docs/evals/2026-08-05-praxis-reach.md b/docs/evals/2026-08-05-praxis-reach.md new file mode 100644 index 0000000..c91d450 --- /dev/null +++ b/docs/evals/2026-08-05-praxis-reach.md @@ -0,0 +1,63 @@ +# Praxis reach at stage 0, 2026-08-05 + +Vikunja #516. Measured with `make eval-reach` on the held-out ecosystem fixture +(`internal/router/eval/ru_ecosystem_v1.json`, 30 cases), classifier + ONNX embedder, +no llama-server in the run. The LLM arm was not measured, so judge a cascade +number again before quoting one. + +## Result + +| | before | after | +|---|---|---| +| overall | 16/30 (53.3%) | 27/30 (90.0%) | +| by want: praxis | 0/12 | 11/12 | +| by want: hexis | 9/10 | 9/10 | +| by want: none | 7/8 | 7/8 | +| by tag: lifecycle | 0/5 | 5/5 | +| by tag: attention | 0/7 | 6/7 | +| by tag: reading | 0/7 | 6/7 | +| wrong praxis arm | 0 | 0 | +| p50 latency | 20.6ms | 16.5ms | + +## Why it was zero + +Not a tuning gap. `handlePraxisAct` dispatches on exact equality between +`Slots.Fn` and a capability alias, and the fn slot is filled by `DefaultActMatcher` +from the deployment's enabled tool names. No Praxis alias is on that list, so no +utterance could put one in the slot. The Russian aliases in `praxisCapabilities` +read as if they matched speech. They are compared against a fn slot and never +against an utterance. + +`PraxisGrammars()` (`internal/router/praxis.go`) fills the slot at stage 0, wired in +`buildRouter` before the capture marker because "отметь" is a capture verb. + +## The three misses that remain + +- `eco-ru-006` "запусти бэкап на нексусе", a Hexis case, routed note. Pre-existing. +- `eco-ru-028` "выключи", reached Hexis, should have asked. Pre-existing. +- `eco-ru-021` "что там с нексусом" wants scoped attention. Deliberately not + claimed. "что там с X" also opens "что там с погодой". Routing a weather + question to Nexus is worse than one missed fixture case. + +## Two judgement calls worth re-arguing + +**A lifecycle word alone does not transition an item.** "готово" is what he says +about the thing he just finished. So the rules split lifecycle words by mood. An +imperative he says to her ("закрывай") claims the turn bare, and the capability +asks which пункт. A stative ("готово", "принято") needs an item named beside it. + +The bare-imperative arm also requires that nothing else in the sentence is being +acted on. "закрой шторы в комнате" is an imperative too. Without that guard it took +a house command to Praxis, measured at hexis 8/10 mid-change. + +**A demonstrative resolves only against a one-item digest.** "отметь это как +сделанное" points at what she just read. `resolveSurfacedPosition` maps it to an id +only when exactly one item was spoken. With two or more it gives the turn back to +the cascade rather than transitioning one of them at random. With no digest at all +it gives the turn back too, because "я это сделал" was never about a пункт. + +## Routing fixture + +`make eval-router`, same run: classifier + ONNX 60/84 (71.4% full and intent-only), +0 false clarifies, 6 missed clarifies (the known `amb-*` set). No failure in that +list comes from a stage-0 decision. Every one carries a classifier confidence score.