Merge the decision trace (#209)
V-564. One decision.Record per turn: the utterance, the winner, and a Claim per claimant carrying its stage, name, the intent it would have made the turn, the score it reported, the outcome and the reason. HasScore is separate from the score so a real 0.0 is not read as no score. Outcomes are won, declined, lost_on_order, lost_on_score, thinned, merged, never_asked. Every stage declares its roster up front, so Finish names everyone who never reported. NEVER ASKED is explicit rather than an absence, which is the fact the hardcoded ordering hides. Covered: the seven pre-route resolvers, eleven stage 0 grammar sets, the LLM router and the classifier with which arm of gateLLMDecision thinned a route, the classifier runners-up, the follow-up merge, 27 query sources, and a terminal action-handler or clarify-ask claim. On by default, no flag. It rides the context like querysource.go and is installed in runTurn, so mic, telegram and web leave the same trail. Storage is a 25-turn in-memory ring: no write on the answer path, no migration, and none of his words outlive the diagnosis. Readable on /trace. TestRecordingDoesNotChangeTheReply answers the same utterances with and without the ring.
This commit is contained in:
@@ -0,0 +1,185 @@
|
||||
// Package decision records who claimed one turn and who lost it (V-564).
|
||||
//
|
||||
// Arbitration between the claimants on the utterance stream is order, hardcoded
|
||||
// in the resolver ladder, in buildRouter and in querySources (V-558). Order is
|
||||
// invisible in a log: the daemon says which intent won and which query source
|
||||
// answered, never who else wanted the turn, with what score, or why it did not
|
||||
// get it. The Rome misroute took a probe, a log read and a code read to explain,
|
||||
// which is one diagnosis too many for a defect family already three deep.
|
||||
//
|
||||
// The record rides the context, the same seam querysource.go uses and for the
|
||||
// same reason: a turn answers through one string that the mic, telegram and the
|
||||
// web all share, so a second return value is not threadable. A context with no
|
||||
// recorder notes nothing, so every Note here is free in a test or a tool that
|
||||
// did not ask for one.
|
||||
//
|
||||
// The most important thing it holds is not a loss but a silence. A claimant
|
||||
// that was NEVER ASKED — because something earlier in the ladder returned
|
||||
// first — looks identical to one that examined the turn and declined, and it is
|
||||
// that confusion the hardcoded ordering hides. So a stage declares its roster
|
||||
// up front and Finish names everyone who never reported.
|
||||
package decision
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Stages, in the order a turn passes through them.
|
||||
const (
|
||||
StagePreRoute = "pre-route" // clarify, confirm, repair and their siblings
|
||||
StageZero = "stage0" // grammar rules
|
||||
StageRoute = "route" // LLM router, classifier
|
||||
StageMerge = "merge" // the follow-up merge, which edits rather than claims
|
||||
StageQuery = "query" // the query source chain
|
||||
StageAction = "action" // whoever actually produced the reply
|
||||
)
|
||||
|
||||
// Outcomes. Coarse on purpose: the record answers "who wanted this turn and
|
||||
// what happened to their claim", not "re-derive the branch".
|
||||
const (
|
||||
Won = "won" // this claimant produced the turn
|
||||
Declined = "declined" // it looked at the turn and said not mine
|
||||
LostOnOrder = "lost_on_order" // it wanted the turn, something earlier had it
|
||||
LostOnScore = "lost_on_score" // it was scored against a rival and scored lower
|
||||
Thinned = "thinned" // it claimed, and a gate cut its confidence
|
||||
Merged = "merged" // it changed the winning claim without owning it
|
||||
NeverAsked = "never_asked" // it never got to look at all
|
||||
)
|
||||
|
||||
// Claim is one claimant's say on one turn.
|
||||
type Claim struct {
|
||||
Stage string `json:"stage"`
|
||||
Claimant string `json:"claimant"`
|
||||
Intent string `json:"intent,omitempty"` // what it would have made the turn
|
||||
Score float64 `json:"score,omitempty"` // only meaningful with HasScore
|
||||
HasScore bool `json:"has_score,omitempty"`
|
||||
Outcome string `json:"outcome"`
|
||||
Reason string `json:"reason,omitempty"` // why it lost, in its own terms
|
||||
}
|
||||
|
||||
// Record is one turn's arbitration. Utterance is held because a record with no
|
||||
// utterance is unreadable, and this store is diagnostics with a short life —
|
||||
// unlike the facts table, which is the audit trail.
|
||||
type Record struct {
|
||||
Ts time.Time `json:"ts"`
|
||||
Utterance string `json:"utterance"`
|
||||
Winner string `json:"winner"`
|
||||
Claims []Claim `json:"claims"`
|
||||
|
||||
mu sync.Mutex
|
||||
rosters []roster
|
||||
}
|
||||
|
||||
type roster struct {
|
||||
stage string
|
||||
names []string
|
||||
}
|
||||
|
||||
// Expect declares the claimants a stage could have asked, so Finish can tell a
|
||||
// decline from a silence. The slice is held, not copied: every caller passes a
|
||||
// package-level table.
|
||||
func (r *Record) Expect(stage string, names []string) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.rosters = append(r.rosters, roster{stage: stage, names: names})
|
||||
}
|
||||
|
||||
// Note appends one claim. The winner is whoever noted Won last, which is the
|
||||
// claimant that actually returned the reply.
|
||||
func (r *Record) Note(c Claim) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.Claims = append(r.Claims, c)
|
||||
if c.Outcome == Won {
|
||||
r.Winner = c.Stage + ":" + c.Claimant
|
||||
}
|
||||
}
|
||||
|
||||
// NoteIfUnclaimed records a win only when nobody has claimed the turn yet. It
|
||||
// is what closes a record whose route was decided but whose reply came from
|
||||
// somewhere with no scoreboard — a clarify question, or an action handler with
|
||||
// no chain in front of it. Without it a thinned route leaves the record with no
|
||||
// winner at all, which reads as a lost turn rather than an asked question.
|
||||
func (r *Record) NoteIfUnclaimed(c Claim) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
r.mu.Lock()
|
||||
claimed := r.Winner != ""
|
||||
r.mu.Unlock()
|
||||
if claimed {
|
||||
return
|
||||
}
|
||||
c.Outcome = Won
|
||||
r.Note(c)
|
||||
}
|
||||
|
||||
// Finish fills in the never-asked claimants and returns the record. Called once
|
||||
// by whoever installed the recorder, after the turn has answered.
|
||||
func (r *Record) Finish(now time.Time) *Record {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.Ts = now
|
||||
reported := map[string]bool{}
|
||||
for _, c := range r.Claims {
|
||||
reported[c.Stage+":"+c.Claimant] = true
|
||||
}
|
||||
for _, ros := range r.rosters {
|
||||
for _, name := range ros.names {
|
||||
if !reported[ros.stage+":"+name] {
|
||||
r.Claims = append(r.Claims, Claim{
|
||||
Stage: ros.stage, Claimant: name, Outcome: NeverAsked,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// --- the context seam ---
|
||||
|
||||
type recorderKey struct{}
|
||||
|
||||
// With returns a context carrying a fresh record, and the record to read after
|
||||
// the turn has answered.
|
||||
func With(ctx context.Context, utterance string) (context.Context, *Record) {
|
||||
rec := &Record{Utterance: utterance}
|
||||
return context.WithValue(ctx, recorderKey{}, rec), rec
|
||||
}
|
||||
|
||||
// From returns the record on the context, or nil. Every method on *Record is
|
||||
// nil-safe, so a caller does not have to check.
|
||||
func From(ctx context.Context) *Record {
|
||||
rec, _ := ctx.Value(recorderKey{}).(*Record)
|
||||
return rec
|
||||
}
|
||||
|
||||
// Note is the shorthand every claim site uses: a no-op when nobody is recording.
|
||||
func Note(ctx context.Context, c Claim) {
|
||||
From(ctx).Note(c)
|
||||
}
|
||||
|
||||
// Expect is the roster shorthand, likewise a no-op with no recorder.
|
||||
func Expect(ctx context.Context, stage string, names []string) {
|
||||
From(ctx).Expect(stage, names)
|
||||
}
|
||||
|
||||
// Scored is a claim carrying a confidence, kept as a constructor so a caller
|
||||
// cannot forget HasScore and have a real 0.0 read as "no score".
|
||||
func Scored(stage, claimant, intent string, score float64, outcome, reason string) Claim {
|
||||
return Claim{
|
||||
Stage: stage, Claimant: claimant, Intent: intent,
|
||||
Score: score, HasScore: true, Outcome: outcome, Reason: reason,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package decision
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestFinishNamesTheNeverAsked(t *testing.T) {
|
||||
ctx, rec := With(context.Background(), "какая погода в риме?")
|
||||
Expect(ctx, StageQuery, []string{"calendar", "weather", "search", "kiwix"})
|
||||
Note(ctx, Claim{Stage: StageQuery, Claimant: "calendar", Outcome: Declined})
|
||||
Note(ctx, Claim{Stage: StageQuery, Claimant: "weather", Outcome: Won})
|
||||
|
||||
rec.Finish(time.Now())
|
||||
|
||||
outcomes := map[string]string{}
|
||||
for _, c := range rec.Claims {
|
||||
outcomes[c.Claimant] = c.Outcome
|
||||
}
|
||||
if outcomes["weather"] != Won || rec.Winner != StageQuery+":weather" {
|
||||
t.Errorf("winner = %q, weather = %q", rec.Winner, outcomes["weather"])
|
||||
}
|
||||
if outcomes["calendar"] != Declined {
|
||||
t.Errorf("calendar = %q, want a decline", outcomes["calendar"])
|
||||
}
|
||||
// The two below the winner never looked, and saying so is the whole point.
|
||||
for _, name := range []string{"search", "kiwix"} {
|
||||
if outcomes[name] != NeverAsked {
|
||||
t.Errorf("%s = %q, want %q", name, outcomes[name], NeverAsked)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A real 0.0 confidence must not read as "this claimant has no score".
|
||||
func TestScoredKeepsAZeroScore(t *testing.T) {
|
||||
c := Scored(StageRoute, "classifier", "chat", 0, LostOnScore, "")
|
||||
if !c.HasScore || c.Score != 0 {
|
||||
t.Errorf("claim = %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoteIfUnclaimedYieldsToARealWinner(t *testing.T) {
|
||||
ctx, rec := With(context.Background(), "x")
|
||||
Note(ctx, Claim{Stage: StageQuery, Claimant: "weather", Outcome: Won})
|
||||
rec.NoteIfUnclaimed(Claim{Stage: StageAction, Claimant: "action-handler"})
|
||||
if rec.Winner != StageQuery+":weather" {
|
||||
t.Errorf("winner = %q, want the query source", rec.Winner)
|
||||
}
|
||||
}
|
||||
|
||||
// A context with no record must cost nothing and crash nothing: that is what
|
||||
// makes the claim sites safe to leave in every test and every fixture run.
|
||||
func TestNoRecorderIsANoOp(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
Note(ctx, Claim{Claimant: "x", Outcome: Won})
|
||||
Expect(ctx, StageQuery, []string{"y"})
|
||||
if From(ctx) != nil {
|
||||
t.Error("bare context reported a record")
|
||||
}
|
||||
From(ctx).NoteIfUnclaimed(Claim{Claimant: "z"})
|
||||
if rec := From(ctx).Finish(time.Now()); rec != nil {
|
||||
t.Error("finishing a nil record produced one")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRingIsBoundedAndNewestFirst(t *testing.T) {
|
||||
r := NewRing()
|
||||
for i := 0; i < ringSize+5; i++ {
|
||||
r.Push(&Record{Utterance: string(rune('a' + i))})
|
||||
}
|
||||
got := r.Recent(ringSize + 10)
|
||||
if len(got) != ringSize {
|
||||
t.Fatalf("kept %d records, want %d", len(got), ringSize)
|
||||
}
|
||||
if got[0].Utterance != string(rune('a'+ringSize+4)) {
|
||||
t.Errorf("newest = %q", got[0].Utterance)
|
||||
}
|
||||
}
|
||||
|
||||
// A query source may fan out to goroutines of its own, so two of them noting at
|
||||
// once must not race. Run under -race, which is where this earns its keep.
|
||||
func TestConcurrentNotes(t *testing.T) {
|
||||
ctx, rec := With(context.Background(), "x")
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 8; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
Note(ctx, Claim{Stage: StageQuery, Claimant: "fanout", Outcome: Declined})
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if len(rec.Claims) != 8 {
|
||||
t.Errorf("recorded %d claims, want 8", len(rec.Claims))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package decision
|
||||
|
||||
import "sync"
|
||||
|
||||
// ringSize is how many turns are kept. Turns arrive at human rate, not machine
|
||||
// rate, so the whole store is memory: no migration, no insert on the answer
|
||||
// path, and nothing of his words survives a restart. That is what makes this
|
||||
// cheap enough to leave on always (V-564). Ecosystem traces went to SQLite
|
||||
// because one act writes several hops and they must outlive the turn; an
|
||||
// arbitration record is read minutes later or never.
|
||||
const ringSize = 25
|
||||
|
||||
// Ring holds the newest records, newest first on read.
|
||||
type Ring struct {
|
||||
mu sync.Mutex
|
||||
recs []*Record
|
||||
}
|
||||
|
||||
func NewRing() *Ring { return &Ring{} }
|
||||
|
||||
// Push adds one finished record and drops the oldest past the bound.
|
||||
func (r *Ring) Push(rec *Record) {
|
||||
if r == nil || rec == nil {
|
||||
return
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.recs = append(r.recs, rec)
|
||||
if len(r.recs) > ringSize {
|
||||
r.recs = r.recs[len(r.recs)-ringSize:]
|
||||
}
|
||||
}
|
||||
|
||||
// Recent returns up to n records, newest first.
|
||||
func (r *Ring) Recent(n int) []*Record {
|
||||
if r == nil || n <= 0 {
|
||||
return nil
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if n > len(r.recs) {
|
||||
n = len(r.recs)
|
||||
}
|
||||
out := make([]*Record, 0, n)
|
||||
for i := 0; i < n; i++ {
|
||||
out = append(out, r.recs[len(r.recs)-1-i])
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -836,6 +836,30 @@ type TickTrace struct {
|
||||
Rules []RuleTrace `json:"rules"`
|
||||
}
|
||||
|
||||
// --- Turn decision trace DTOs (V-564) ---
|
||||
|
||||
// TurnClaim — one claimant's say on one turn: who, at which stage, what it
|
||||
// would have made the turn, the score it reported if it has one, and what
|
||||
// happened to the claim. Same shape as RuleTrace above and for the same reason:
|
||||
// a winner alone does not explain an arbitration, the losers do.
|
||||
type TurnClaim struct {
|
||||
Stage string `json:"stage"`
|
||||
Claimant string `json:"claimant"`
|
||||
Intent string `json:"intent,omitempty"`
|
||||
Score float64 `json:"score,omitempty"`
|
||||
HasScore bool `json:"has_score,omitempty"`
|
||||
Outcome string `json:"outcome"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
// TurnDecision — one turn's arbitration, newest first when read as a list.
|
||||
type TurnDecision struct {
|
||||
Ts time.Time `json:"ts"`
|
||||
Utterance string `json:"utterance"`
|
||||
Winner string `json:"winner"`
|
||||
Claims []TurnClaim `json:"claims"`
|
||||
}
|
||||
|
||||
// MorningRoutineItem — one checklist entry's current state.
|
||||
type MorningRoutineItem struct {
|
||||
Key string `json:"key"`
|
||||
|
||||
@@ -672,6 +672,14 @@ func (c *Client) TickTrace(ctx context.Context) (TickTrace, error) {
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func (c *Client) TurnDecisions(ctx context.Context, n int) ([]TurnDecision, error) {
|
||||
var d []TurnDecision
|
||||
if err := c.call(ctx, MethodTurnDecisions, nReq{N: n}, &d); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func (c *Client) RecentEvents(ctx context.Context, n int) ([]IntakeEvent, error) {
|
||||
var e []IntakeEvent
|
||||
if err := c.call(ctx, MethodRecentEvents, nReq{N: n}, &e); err != nil {
|
||||
|
||||
@@ -169,6 +169,13 @@ type SystemAPI interface {
|
||||
// persisted — it's a daemon-level cache).
|
||||
TickTrace(ctx context.Context) (TickTrace, error)
|
||||
|
||||
// TurnDecisions returns the newest turn arbitration records, newest first
|
||||
// (V-564). Same shape as TickTrace and RecentEvents: a bounded in-memory
|
||||
// ring on the daemon, so the store adapter returns an error rather than
|
||||
// pretending a table exists. Empty is a normal answer — it means no turn
|
||||
// has run since the daemon started.
|
||||
TurnDecisions(ctx context.Context, n int) ([]TurnDecision, error)
|
||||
|
||||
// RecentEcosystemTraces reads the ecosystem call log, which lives in its
|
||||
// own table so machine-rate traces never crowd out human-rate facts.
|
||||
RecentEcosystemTraces(ctx context.Context, n int) ([]EcosystemTrace, error)
|
||||
|
||||
@@ -583,6 +583,13 @@ var methodTable = map[Method]handlerFunc{
|
||||
MethodTickTrace: withoutParams(func(ctx context.Context, api CoreAPI) (TickTrace, error) {
|
||||
return api.TickTrace(ctx)
|
||||
}),
|
||||
MethodTurnDecisions: withParams(func(ctx context.Context, api CoreAPI, p nReq) ([]TurnDecision, error) {
|
||||
d, err := api.TurnDecisions(ctx, p.N)
|
||||
if d == nil {
|
||||
d = []TurnDecision{}
|
||||
}
|
||||
return d, err
|
||||
}),
|
||||
// MorningStatus intentionally has no nil→[]T{} normalization here — the
|
||||
// pre-table arm marshaled api.MorningStatus's result as-is (a nil slice
|
||||
// serializes as JSON null), and this preserves that exact wire shape.
|
||||
|
||||
@@ -265,6 +265,12 @@ func (a *storeAPI) TickTrace(ctx context.Context) (TickTrace, error) {
|
||||
return TickTrace{}, errors.New("store: tick trace not available via direct store API")
|
||||
}
|
||||
|
||||
// TurnDecisions — same story as TickTrace: the arbitration record is a daemon
|
||||
// ring, not a table, so there is nothing here to read it from (V-564).
|
||||
func (a *storeAPI) TurnDecisions(ctx context.Context, n int) ([]TurnDecision, error) {
|
||||
return nil, errors.New("store: turn decisions not available via direct store API")
|
||||
}
|
||||
|
||||
// SeedEvent — same shape as MorningStatus: writing the fact is a store call,
|
||||
// but extraction and detect-and-propose live in mavend, and a seed that wrote
|
||||
// the fact without running them would be the one thing this seam must not be,
|
||||
|
||||
@@ -144,6 +144,9 @@ func (UnimplementedCoreAPI) RevertFact(ctx context.Context, key string) (int64,
|
||||
func (UnimplementedCoreAPI) TickTrace(ctx context.Context) (TickTrace, error) {
|
||||
return TickTrace{}, ErrNotImplemented
|
||||
}
|
||||
func (UnimplementedCoreAPI) TurnDecisions(ctx context.Context, n int) ([]TurnDecision, error) {
|
||||
return nil, ErrNotImplemented
|
||||
}
|
||||
func (UnimplementedCoreAPI) MorningStatus(ctx context.Context) ([]MorningRoutineStatus, error) {
|
||||
return nil, ErrNotImplemented
|
||||
}
|
||||
|
||||
@@ -48,6 +48,7 @@ const (
|
||||
MethodAcceptProposedRoutine Method = "accept_proposed_routine"
|
||||
MethodRevertFact Method = "revert_fact"
|
||||
MethodTickTrace Method = "tick_trace"
|
||||
MethodTurnDecisions Method = "turn_decisions"
|
||||
MethodMorningStatus Method = "morning_status"
|
||||
MethodMCPServers Method = "mcp_servers"
|
||||
MethodDayPlan Method = "day_plan"
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
// router/decisiontrace.go — what the cascade tells the per-turn decision record.
|
||||
//
|
||||
// The cascade's arbitration is order (V-558): the first grammar whose Build
|
||||
// agrees wins, and the model and the classifier are only reached because nobody
|
||||
// upstream did. None of that is visible afterwards, so V-564 has each stage say
|
||||
// its piece into the record riding the context. Nothing here reads the record
|
||||
// back and nothing here can change a route — a nil recorder is the normal case
|
||||
// in the fixture runner and every router test.
|
||||
package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/kami/maven/internal/decision"
|
||||
)
|
||||
|
||||
// The two routing engines, named as claimants. They are one stage and not two,
|
||||
// because only one of them ever runs: the classifier is reached when the model
|
||||
// is absent or errored, never alongside it.
|
||||
const (
|
||||
claimantLLM = "llm-router"
|
||||
claimantClassifier = "classifier"
|
||||
)
|
||||
|
||||
// thinReason names which arm of gateLLMDecision cut the confidence. The gate
|
||||
// has three structural holes and they are three different defects, so "thinned"
|
||||
// alone is not enough to act on.
|
||||
func thinReason(d *Decision) string {
|
||||
switch {
|
||||
case d.Intent == IntentFact && !d.Slots.HasKey:
|
||||
return "a fact with no key even after the parser tried"
|
||||
case d.Intent == IntentAct && !d.Slots.HasFn:
|
||||
return "an act that never resolved to an allowlisted fn"
|
||||
case d.Intent == IntentReminder && !reminderHasSubject(d.Slots.Text):
|
||||
return "a reminder with no subject to say at the hour"
|
||||
default:
|
||||
return "below the clarify threshold"
|
||||
}
|
||||
}
|
||||
|
||||
// Reasons a stage-0 grammar did not take a turn. Kept apart because they are
|
||||
// different defects: a pattern that never matched is a rule that does not know
|
||||
// the shape, a Build that declined is a rule that knew the shape and refused
|
||||
// the content (narrative-query and the wakeword acts do this by design), and a
|
||||
// grammar after the winner was never consulted at all.
|
||||
const (
|
||||
reasonNoMatch = "pattern did not match"
|
||||
reasonBuildDeclmn = "matched the shape, Build declined the content"
|
||||
reasonEarlierClaim = "an earlier grammar claimed the turn"
|
||||
)
|
||||
|
||||
// noteGrammarOutcomes records the stage-0 pass. examined is how many grammars
|
||||
// were reached; declined holds the names whose Build said no; won is the winner
|
||||
// or empty. Everything past the winner is named as never asked, because that
|
||||
// silence is the thing the hardcoded order hides.
|
||||
func (r *Router) noteGrammarOutcomes(ctx context.Context, examined int, declined map[int]bool, won string, intent Intent) {
|
||||
rec := decision.From(ctx)
|
||||
if rec == nil {
|
||||
return
|
||||
}
|
||||
for i, g := range r.grammars {
|
||||
switch {
|
||||
case i >= examined:
|
||||
rec.Note(decision.Claim{
|
||||
Stage: decision.StageZero, Claimant: g.Name,
|
||||
Outcome: decision.NeverAsked, Reason: reasonEarlierClaim,
|
||||
})
|
||||
case g.Name == won:
|
||||
rec.Note(decision.Scored(decision.StageZero, g.Name, string(intent), 1.0,
|
||||
decision.Won, ""))
|
||||
case declined[i]:
|
||||
rec.Note(decision.Claim{
|
||||
Stage: decision.StageZero, Claimant: g.Name,
|
||||
Outcome: decision.Declined, Reason: reasonBuildDeclmn,
|
||||
})
|
||||
default:
|
||||
rec.Note(decision.Claim{
|
||||
Stage: decision.StageZero, Claimant: g.Name,
|
||||
Outcome: decision.Declined, Reason: reasonNoMatch,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"errors"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/decision"
|
||||
)
|
||||
|
||||
// Config — wires the cascade. Build via New; a zero-value Router is unusable.
|
||||
@@ -67,7 +69,11 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De
|
||||
// the STT often includes one (transcribed phonetically, any script) — try
|
||||
// the wake-stripped utterance too so those grammars still fire.
|
||||
stripped, hadWake := StripWakeToken(utterance)
|
||||
for _, g := range r.grammars {
|
||||
// declinedBuild — the grammars that matched the shape and refused the
|
||||
// content, kept for the decision record (V-564) so a reader can tell that
|
||||
// rule from one whose pattern never fired.
|
||||
var declinedBuild map[int]bool
|
||||
for i, g := range r.grammars {
|
||||
m := g.Pattern.FindStringSubmatch(utterance)
|
||||
if m == nil && hadWake {
|
||||
m = g.Pattern.FindStringSubmatch(stripped)
|
||||
@@ -77,11 +83,17 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De
|
||||
}
|
||||
d, ok := g.Build(m)
|
||||
if !ok {
|
||||
if declinedBuild == nil {
|
||||
declinedBuild = map[int]bool{}
|
||||
}
|
||||
declinedBuild[i] = true
|
||||
continue // grammar matched shape but not content → fall through
|
||||
}
|
||||
d.Utterance = utterance
|
||||
r.noteGrammarOutcomes(ctx, i+1, declinedBuild, g.Name, d.Intent)
|
||||
return d, nil
|
||||
}
|
||||
r.noteGrammarOutcomes(ctx, len(r.grammars), declinedBuild, "", "")
|
||||
|
||||
// stage 1a — LLM router (when wired). It reasons over the utterance instead
|
||||
// of nearest-centroid guessing. On any error/parse-fail, fall through to the
|
||||
@@ -90,11 +102,38 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De
|
||||
if d, ok, err := r.llm.Route(ctx, utterance, now); err == nil && ok {
|
||||
d.Utterance = utterance
|
||||
r.fillSlots(ctx, &d, now)
|
||||
before := d.Confidence
|
||||
r.gateLLMDecision(&d)
|
||||
// The classifier is the floor and it never ran, which is the whole
|
||||
// reason a wrong LLM route reads as unexplainable (V-564).
|
||||
decision.Note(ctx, decision.Claim{
|
||||
Stage: decision.StageRoute, Claimant: claimantClassifier,
|
||||
Outcome: decision.NeverAsked, Reason: "the LLM router answered",
|
||||
})
|
||||
outcome, reason := decision.Won, ""
|
||||
if d.Confidence < before {
|
||||
outcome, reason = decision.Thinned, thinReason(&d)
|
||||
}
|
||||
decision.Note(ctx, decision.Scored(decision.StageRoute, claimantLLM,
|
||||
string(d.Intent), d.Confidence, outcome, reason))
|
||||
return d, nil
|
||||
} else if err != nil {
|
||||
log.Printf("router: llm route fell back to classifier: %v", err)
|
||||
decision.Note(ctx, decision.Claim{
|
||||
Stage: decision.StageRoute, Claimant: claimantLLM,
|
||||
Outcome: decision.Declined, Reason: "error: " + err.Error(),
|
||||
})
|
||||
} else {
|
||||
decision.Note(ctx, decision.Claim{
|
||||
Stage: decision.StageRoute, Claimant: claimantLLM,
|
||||
Outcome: decision.Declined, Reason: "no parsable route in the reply",
|
||||
})
|
||||
}
|
||||
} else {
|
||||
decision.Note(ctx, decision.Claim{
|
||||
Stage: decision.StageRoute, Claimant: claimantLLM,
|
||||
Outcome: decision.NeverAsked, Reason: "no LLM router is wired",
|
||||
})
|
||||
}
|
||||
|
||||
// stage 1 — intent classifier.
|
||||
@@ -103,6 +142,16 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De
|
||||
return Decision{}, err
|
||||
}
|
||||
best := results[0]
|
||||
// The runners-up are the interesting part: two intents a hundredth apart is
|
||||
// a different defect from one that won outright (V-564). Two are enough to
|
||||
// see that, and the rest of a seven-intent scoreboard is noise on the page.
|
||||
if rec := decision.From(ctx); rec != nil {
|
||||
for _, res := range results[1:min(len(results), 3)] {
|
||||
rec.Note(decision.Scored(decision.StageRoute, claimantClassifier,
|
||||
string(res.Intent), res.Score, decision.LostOnScore,
|
||||
"lower similarity than "+string(best.Intent)))
|
||||
}
|
||||
}
|
||||
|
||||
// stage 2 — slot extraction for the winning intent.
|
||||
d := Decision{
|
||||
@@ -118,6 +167,12 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De
|
||||
d.Stage = 3
|
||||
d.Clarify = true
|
||||
}
|
||||
outcome, reason := decision.Won, ""
|
||||
if d.Clarify {
|
||||
outcome, reason = decision.Thinned, "below the clarify threshold, so she asks instead"
|
||||
}
|
||||
decision.Note(ctx, decision.Scored(decision.StageRoute, claimantClassifier,
|
||||
string(d.Intent), d.Confidence, outcome, reason))
|
||||
return d, nil
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user