From 5417692566df53048ef4a97d311818e80f6df4dd Mon Sep 17 00:00:00 2001 From: claude Date: Thu, 6 Aug 2026 00:52:28 +0400 Subject: [PATCH] the cascade says which grammar declined and which never ran (V-564) Stage 0 records every grammar it reached, keeping a pattern that never matched apart from a Build that refused the content, and names the ones after the winner as never asked. The routing arm records the classifier's runners-up and which arm of gateLLMDecision cut the confidence, because thinned alone is not enough to act on. --- internal/router/decisiontrace.go | 83 ++++++++++++++++++++++++++++++++ internal/router/router.go | 57 +++++++++++++++++++++- 2 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 internal/router/decisiontrace.go diff --git a/internal/router/decisiontrace.go b/internal/router/decisiontrace.go new file mode 100644 index 0000000..f66ac2a --- /dev/null +++ b/internal/router/decisiontrace.go @@ -0,0 +1,83 @@ +// router/decisiontrace.go — what the cascade tells the per-turn decision record. +// +// The cascade's arbitration is order (V-558): the first grammar whose Build +// agrees wins, and the model and the classifier are only reached because nobody +// upstream did. None of that is visible afterwards, so V-564 has each stage say +// its piece into the record riding the context. Nothing here reads the record +// back and nothing here can change a route — a nil recorder is the normal case +// in the fixture runner and every router test. +package router + +import ( + "context" + + "github.com/kami/maven/internal/decision" +) + +// The two routing engines, named as claimants. They are one stage and not two, +// because only one of them ever runs: the classifier is reached when the model +// is absent or errored, never alongside it. +const ( + claimantLLM = "llm-router" + claimantClassifier = "classifier" +) + +// thinReason names which arm of gateLLMDecision cut the confidence. The gate +// has three structural holes and they are three different defects, so "thinned" +// alone is not enough to act on. +func thinReason(d *Decision) string { + switch { + case d.Intent == IntentFact && !d.Slots.HasKey: + return "a fact with no key even after the parser tried" + case d.Intent == IntentAct && !d.Slots.HasFn: + return "an act that never resolved to an allowlisted fn" + case d.Intent == IntentReminder && !reminderHasSubject(d.Slots.Text): + return "a reminder with no subject to say at the hour" + default: + return "below the clarify threshold" + } +} + +// Reasons a stage-0 grammar did not take a turn. Kept apart because they are +// different defects: a pattern that never matched is a rule that does not know +// the shape, a Build that declined is a rule that knew the shape and refused +// the content (narrative-query and the wakeword acts do this by design), and a +// grammar after the winner was never consulted at all. +const ( + reasonNoMatch = "pattern did not match" + reasonBuildDeclmn = "matched the shape, Build declined the content" + reasonEarlierClaim = "an earlier grammar claimed the turn" +) + +// noteGrammarOutcomes records the stage-0 pass. examined is how many grammars +// were reached; declined holds the names whose Build said no; won is the winner +// or empty. Everything past the winner is named as never asked, because that +// silence is the thing the hardcoded order hides. +func (r *Router) noteGrammarOutcomes(ctx context.Context, examined int, declined map[int]bool, won string, intent Intent) { + rec := decision.From(ctx) + if rec == nil { + return + } + for i, g := range r.grammars { + switch { + case i >= examined: + rec.Note(decision.Claim{ + Stage: decision.StageZero, Claimant: g.Name, + Outcome: decision.NeverAsked, Reason: reasonEarlierClaim, + }) + case g.Name == won: + rec.Note(decision.Scored(decision.StageZero, g.Name, string(intent), 1.0, + decision.Won, "")) + case declined[i]: + rec.Note(decision.Claim{ + Stage: decision.StageZero, Claimant: g.Name, + Outcome: decision.Declined, Reason: reasonBuildDeclmn, + }) + default: + rec.Note(decision.Claim{ + Stage: decision.StageZero, Claimant: g.Name, + Outcome: decision.Declined, Reason: reasonNoMatch, + }) + } + } +} diff --git a/internal/router/router.go b/internal/router/router.go index 68457e3..d8c8088 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -5,6 +5,8 @@ import ( "errors" "log" "time" + + "github.com/kami/maven/internal/decision" ) // Config — wires the cascade. Build via New; a zero-value Router is unusable. @@ -67,7 +69,11 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De // the STT often includes one (transcribed phonetically, any script) — try // the wake-stripped utterance too so those grammars still fire. stripped, hadWake := StripWakeToken(utterance) - for _, g := range r.grammars { + // declinedBuild — the grammars that matched the shape and refused the + // content, kept for the decision record (V-564) so a reader can tell that + // rule from one whose pattern never fired. + var declinedBuild map[int]bool + for i, g := range r.grammars { m := g.Pattern.FindStringSubmatch(utterance) if m == nil && hadWake { m = g.Pattern.FindStringSubmatch(stripped) @@ -77,11 +83,17 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De } d, ok := g.Build(m) if !ok { + if declinedBuild == nil { + declinedBuild = map[int]bool{} + } + declinedBuild[i] = true continue // grammar matched shape but not content → fall through } d.Utterance = utterance + r.noteGrammarOutcomes(ctx, i+1, declinedBuild, g.Name, d.Intent) return d, nil } + r.noteGrammarOutcomes(ctx, len(r.grammars), declinedBuild, "", "") // stage 1a — LLM router (when wired). It reasons over the utterance instead // of nearest-centroid guessing. On any error/parse-fail, fall through to the @@ -90,11 +102,38 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De if d, ok, err := r.llm.Route(ctx, utterance, now); err == nil && ok { d.Utterance = utterance r.fillSlots(ctx, &d, now) + before := d.Confidence r.gateLLMDecision(&d) + // The classifier is the floor and it never ran, which is the whole + // reason a wrong LLM route reads as unexplainable (V-564). + decision.Note(ctx, decision.Claim{ + Stage: decision.StageRoute, Claimant: claimantClassifier, + Outcome: decision.NeverAsked, Reason: "the LLM router answered", + }) + outcome, reason := decision.Won, "" + if d.Confidence < before { + outcome, reason = decision.Thinned, thinReason(&d) + } + decision.Note(ctx, decision.Scored(decision.StageRoute, claimantLLM, + string(d.Intent), d.Confidence, outcome, reason)) return d, nil } else if err != nil { log.Printf("router: llm route fell back to classifier: %v", err) + decision.Note(ctx, decision.Claim{ + Stage: decision.StageRoute, Claimant: claimantLLM, + Outcome: decision.Declined, Reason: "error: " + err.Error(), + }) + } else { + decision.Note(ctx, decision.Claim{ + Stage: decision.StageRoute, Claimant: claimantLLM, + Outcome: decision.Declined, Reason: "no parsable route in the reply", + }) } + } else { + decision.Note(ctx, decision.Claim{ + Stage: decision.StageRoute, Claimant: claimantLLM, + Outcome: decision.NeverAsked, Reason: "no LLM router is wired", + }) } // stage 1 — intent classifier. @@ -103,6 +142,16 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De return Decision{}, err } best := results[0] + // The runners-up are the interesting part: two intents a hundredth apart is + // a different defect from one that won outright (V-564). Two are enough to + // see that, and the rest of a seven-intent scoreboard is noise on the page. + if rec := decision.From(ctx); rec != nil { + for _, res := range results[1:min(len(results), 3)] { + rec.Note(decision.Scored(decision.StageRoute, claimantClassifier, + string(res.Intent), res.Score, decision.LostOnScore, + "lower similarity than "+string(best.Intent))) + } + } // stage 2 — slot extraction for the winning intent. d := Decision{ @@ -118,6 +167,12 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De d.Stage = 3 d.Clarify = true } + outcome, reason := decision.Won, "" + if d.Clarify { + outcome, reason = decision.Thinned, "below the clarify threshold, so she asks instead" + } + decision.Note(ctx, decision.Scored(decision.StageRoute, claimantClassifier, + string(d.Intent), d.Confidence, outcome, reason)) return d, nil }