claude
68a3c85186
They run before the model because they are two orders of magnitude faster and score better on both halves of the route. They decline rather than clarify, so a declined turn carries on to the model and then the classifier, which is what a box with no weights file does on every turn. Nil heads are byte-for-byte the cascade that shipped before this. Measured on the 96-case fixture, classifier+ONNX either way: intent 76.0% -> 96.9% destination 36.4% -> 75.8% false clarify 0 -> 1 missed clarify 8 -> 1 p50 24.5ms -> 27.9ms That beats the gemma-4-12b cascade on both halves, 84.4% and 72.7%, at a twelfth of its 329ms. The four remaining destination misses are all calendar, which is the stage 0 trade V-660 flagged and the owner has not called yet. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ptwopxyo3Z2kwFckHkLvN
87 lines
3.3 KiB
Go
87 lines
3.3 KiB
Go
// router/decisiontrace.go — what the cascade tells the per-turn decision record.
|
|
//
|
|
// The cascade's arbitration is order (V-558): the first grammar whose Build
|
|
// agrees wins, and the model and the classifier are only reached because nobody
|
|
// upstream did. None of that is visible afterwards, so V-564 has each stage say
|
|
// its piece into the record riding the context. Nothing here reads the record
|
|
// back and nothing here can change a route — a nil recorder is the normal case
|
|
// in the fixture runner and every router test.
|
|
package router
|
|
|
|
import (
|
|
"context"
|
|
|
|
"github.com/kami/maven/internal/decision"
|
|
)
|
|
|
|
// The three routing engines, named as claimants. The model and the classifier
|
|
// are one stage and not two, because only one of them ever runs: the classifier
|
|
// is reached when the model is absent or errored, never alongside it. The heads
|
|
// run before both and decline on low confidence, so they can appear beside
|
|
// either one in a record.
|
|
const (
|
|
claimantLLM = "llm-router"
|
|
claimantClassifier = "classifier"
|
|
claimantHeads = "routing-heads"
|
|
)
|
|
|
|
// thinReason names which arm of gateLLMDecision cut the confidence. The gate
|
|
// has three structural holes and they are three different defects, so "thinned"
|
|
// alone is not enough to act on.
|
|
func thinReason(d *Decision) string {
|
|
switch {
|
|
case d.Intent == IntentFact && !d.Slots.HasKey:
|
|
return "a fact with no key even after the parser tried"
|
|
case d.Intent == IntentAct && !d.Slots.HasFn:
|
|
return "an act that never resolved to an allowlisted fn"
|
|
case d.Intent == IntentReminder && !reminderHasSubject(d.Slots.Text):
|
|
return "a reminder with no subject to say at the hour"
|
|
default:
|
|
return "below the clarify threshold"
|
|
}
|
|
}
|
|
|
|
// Reasons a stage-0 grammar did not take a turn. Kept apart because they are
|
|
// different defects: a pattern that never matched is a rule that does not know
|
|
// the shape, a Build that declined is a rule that knew the shape and refused
|
|
// the content (narrative-query and the wakeword acts do this by design), and a
|
|
// grammar after the winner was never consulted at all.
|
|
const (
|
|
reasonNoMatch = "pattern did not match"
|
|
reasonBuildDeclmn = "matched the shape, Build declined the content"
|
|
reasonEarlierClaim = "an earlier grammar claimed the turn"
|
|
)
|
|
|
|
// noteGrammarOutcomes records the stage-0 pass. examined is how many grammars
|
|
// were reached; declined holds the names whose Build said no; won is the winner
|
|
// or empty. Everything past the winner is named as never asked, because that
|
|
// silence is the thing the hardcoded order hides.
|
|
func (r *Router) noteGrammarOutcomes(ctx context.Context, examined int, declined map[int]bool, won string, intent Intent) {
|
|
rec := decision.From(ctx)
|
|
if rec == nil {
|
|
return
|
|
}
|
|
for i, g := range r.grammars {
|
|
switch {
|
|
case i >= examined:
|
|
rec.Note(decision.Claim{
|
|
Stage: decision.StageZero, Claimant: g.Name,
|
|
Outcome: decision.NeverAsked, Reason: reasonEarlierClaim,
|
|
})
|
|
case g.Name == won:
|
|
rec.Note(decision.Scored(decision.StageZero, g.Name, string(intent), 1.0,
|
|
decision.Won, ""))
|
|
case declined[i]:
|
|
rec.Note(decision.Claim{
|
|
Stage: decision.StageZero, Claimant: g.Name,
|
|
Outcome: decision.Declined, Reason: reasonBuildDeclmn,
|
|
})
|
|
default:
|
|
rec.Note(decision.Claim{
|
|
Stage: decision.StageZero, Claimant: g.Name,
|
|
Outcome: decision.Declined, Reason: reasonNoMatch,
|
|
})
|
|
}
|
|
}
|
|
}
|