the cascade says which grammar declined and which never ran (V-564)

Stage 0 records every grammar it reached, keeping a pattern that never matched apart from a Build that refused the content, and names the ones after the winner as never asked. The routing arm records the classifier's runners-up and which arm of gateLLMDecision cut the confidence, because thinned alone is not enough to act on.
2026-08-06 00:52:28 +04:00
parent b56e0e6248
commit 5417692566
2 changed files with 139 additions and 1 deletions
@@ -0,0 +1,83 @@
+// router/decisiontrace.go — what the cascade tells the per-turn decision record.
+//
+// The cascade's arbitration is order (V-558): the first grammar whose Build
+// agrees wins, and the model and the classifier are only reached because nobody
+// upstream did. None of that is visible afterwards, so V-564 has each stage say
+// its piece into the record riding the context. Nothing here reads the record
+// back and nothing here can change a route — a nil recorder is the normal case
+// in the fixture runner and every router test.
+package router
+
+import (
+	"context"
+
+	"github.com/kami/maven/internal/decision"
+)
+
+// The two routing engines, named as claimants. They are one stage and not two,
+// because only one of them ever runs: the classifier is reached when the model
+// is absent or errored, never alongside it.
+const (
+	claimantLLM        = "llm-router"
+	claimantClassifier = "classifier"
+)
+
+// thinReason names which arm of gateLLMDecision cut the confidence. The gate
+// has three structural holes and they are three different defects, so "thinned"
+// alone is not enough to act on.
+func thinReason(d *Decision) string {
+	switch {
+	case d.Intent == IntentFact && !d.Slots.HasKey:
+		return "a fact with no key even after the parser tried"
+	case d.Intent == IntentAct && !d.Slots.HasFn:
+		return "an act that never resolved to an allowlisted fn"
+	case d.Intent == IntentReminder && !reminderHasSubject(d.Slots.Text):
+		return "a reminder with no subject to say at the hour"
+	default:
+		return "below the clarify threshold"
+	}
+}
+
+// Reasons a stage-0 grammar did not take a turn. Kept apart because they are
+// different defects: a pattern that never matched is a rule that does not know
+// the shape, a Build that declined is a rule that knew the shape and refused
+// the content (narrative-query and the wakeword acts do this by design), and a
+// grammar after the winner was never consulted at all.
+const (
+	reasonNoMatch      = "pattern did not match"
+	reasonBuildDeclmn  = "matched the shape, Build declined the content"
+	reasonEarlierClaim = "an earlier grammar claimed the turn"
+)
+
+// noteGrammarOutcomes records the stage-0 pass. examined is how many grammars
+// were reached; declined holds the names whose Build said no; won is the winner
+// or empty. Everything past the winner is named as never asked, because that
+// silence is the thing the hardcoded order hides.
+func (r *Router) noteGrammarOutcomes(ctx context.Context, examined int, declined map[int]bool, won string, intent Intent) {
+	rec := decision.From(ctx)
+	if rec == nil {
+		return
+	}
+	for i, g := range r.grammars {
+		switch {
+		case i >= examined:
+			rec.Note(decision.Claim{
+				Stage: decision.StageZero, Claimant: g.Name,
+				Outcome: decision.NeverAsked, Reason: reasonEarlierClaim,
+			})
+		case g.Name == won:
+			rec.Note(decision.Scored(decision.StageZero, g.Name, string(intent), 1.0,
+				decision.Won, ""))
+		case declined[i]:
+			rec.Note(decision.Claim{
+				Stage: decision.StageZero, Claimant: g.Name,
+				Outcome: decision.Declined, Reason: reasonBuildDeclmn,
+			})
+		default:
+			rec.Note(decision.Claim{
+				Stage: decision.StageZero, Claimant: g.Name,
+				Outcome: decision.Declined, Reason: reasonNoMatch,
+			})
+		}
+	}
+}
@@ -5,6 +5,8 @@ import (
 	"errors"
 	"log"
 	"time"
+
+	"github.com/kami/maven/internal/decision"
 )

 // Config — wires the cascade. Build via New; a zero-value Router is unusable.
@@ -67,7 +69,11 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De
 	// the STT often includes one (transcribed phonetically, any script) — try
 	// the wake-stripped utterance too so those grammars still fire.
 	stripped, hadWake := StripWakeToken(utterance)
-	for _, g := range r.grammars {
+	// declinedBuild — the grammars that matched the shape and refused the
+	// content, kept for the decision record (V-564) so a reader can tell that
+	// rule from one whose pattern never fired.
+	var declinedBuild map[int]bool
+	for i, g := range r.grammars {
 		m := g.Pattern.FindStringSubmatch(utterance)
 		if m == nil && hadWake {
 			m = g.Pattern.FindStringSubmatch(stripped)
@@ -77,11 +83,17 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De
 		}
 		d, ok := g.Build(m)
 		if !ok {
+			if declinedBuild == nil {
+				declinedBuild = map[int]bool{}
+			}
+			declinedBuild[i] = true
 			continue // grammar matched shape but not content → fall through
 		}
 		d.Utterance = utterance
+		r.noteGrammarOutcomes(ctx, i+1, declinedBuild, g.Name, d.Intent)
 		return d, nil
 	}
+	r.noteGrammarOutcomes(ctx, len(r.grammars), declinedBuild, "", "")

 	// stage 1a — LLM router (when wired). It reasons over the utterance instead
 	// of nearest-centroid guessing. On any error/parse-fail, fall through to the
@@ -90,11 +102,38 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De
 		if d, ok, err := r.llm.Route(ctx, utterance, now); err == nil && ok {
 			d.Utterance = utterance
 			r.fillSlots(ctx, &d, now)
+			before := d.Confidence
 			r.gateLLMDecision(&d)
+			// The classifier is the floor and it never ran, which is the whole
+			// reason a wrong LLM route reads as unexplainable (V-564).
+			decision.Note(ctx, decision.Claim{
+				Stage: decision.StageRoute, Claimant: claimantClassifier,
+				Outcome: decision.NeverAsked, Reason: "the LLM router answered",
+			})
+			outcome, reason := decision.Won, ""
+			if d.Confidence < before {
+				outcome, reason = decision.Thinned, thinReason(&d)
+			}
+			decision.Note(ctx, decision.Scored(decision.StageRoute, claimantLLM,
+				string(d.Intent), d.Confidence, outcome, reason))
 			return d, nil
 		} else if err != nil {
 			log.Printf("router: llm route fell back to classifier: %v", err)
+			decision.Note(ctx, decision.Claim{
+				Stage: decision.StageRoute, Claimant: claimantLLM,
+				Outcome: decision.Declined, Reason: "error: " + err.Error(),
+			})
+		} else {
+			decision.Note(ctx, decision.Claim{
+				Stage: decision.StageRoute, Claimant: claimantLLM,
+				Outcome: decision.Declined, Reason: "no parsable route in the reply",
+			})
 		}
+	} else {
+		decision.Note(ctx, decision.Claim{
+			Stage: decision.StageRoute, Claimant: claimantLLM,
+			Outcome: decision.NeverAsked, Reason: "no LLM router is wired",
+		})
 	}

 	// stage 1 — intent classifier.
@@ -103,6 +142,16 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De
 		return Decision{}, err
 	}
 	best := results[0]
+	// The runners-up are the interesting part: two intents a hundredth apart is
+	// a different defect from one that won outright (V-564). Two are enough to
+	// see that, and the rest of a seven-intent scoreboard is noise on the page.
+	if rec := decision.From(ctx); rec != nil {
+		for _, res := range results[1:min(len(results), 3)] {
+			rec.Note(decision.Scored(decision.StageRoute, claimantClassifier,
+				string(res.Intent), res.Score, decision.LostOnScore,
+				"lower similarity than "+string(best.Intent)))
+		}
+	}

 	// stage 2 — slot extraction for the winning intent.
 	d := Decision{
@@ -118,6 +167,12 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De
 		d.Stage = 3
 		d.Clarify = true
 	}
+	outcome, reason := decision.Won, ""
+	if d.Clarify {
+		outcome, reason = decision.Thinned, "below the clarify threshold, so she asks instead"
+	}
+	decision.Note(ctx, decision.Scored(decision.StageRoute, claimantClassifier,
+		string(d.Intent), d.Confidence, outcome, reason))
 	return d, nil
 }