diff --git a/internal/router/claim.go b/internal/router/claim.go new file mode 100644 index 0000000..cbbb8ed --- /dev/null +++ b/internal/router/claim.go @@ -0,0 +1,113 @@ +package router + +import "github.com/kami/maven/internal/claim" + +// ClaimOf — build a claim.Claim from a Decision (V-565, design in +// docs/plans/19-dialogue-arbitration.md). +// +// Additive and beside the existing path. Decision.Confidence keeps its float +// and keeps working: r.threshold and gateLLMDecision read it, and the +// classifier cascade is the failure floor. Nothing in Route calls this yet. +// The arbiter that reads claims is V-560. +// +// claimant names who produced the decision. The cascade does not record which +// stage-0 grammar matched, so the caller passes what it knows and the builder +// does not guess. +func ClaimOf(claimant string, d Decision) claim.Claim { + consumed, unexplained := claim.Split(d.Utterance, claimSpans(d)...) + return claim.Claim{ + Claimant: claimant, + Intent: string(d.Intent), + Filled: filledSlots(d.Slots), + Consumed: consumed, + Unexplained: unexplained, + Band: bandOf(d), + Veto: vetoOf(d), + } +} + +// claimSpans — the parts of the utterance the decision says it read. Slot +// values, not the utterance, because coverage is the question of how much of +// the sentence the claim actually explains. +// +// A stage-0 grammar reports whatever its Build put in the slots, which for the +// reminder rule is the text after "напомни" and not the verb itself. That +// under-reports coverage rather than over-reporting it, which is the safe +// direction: a claim that overstates what it explains wins arbitrations it +// should lose. +func claimSpans(d Decision) []string { + spans := []string{d.Slots.Text, d.Slots.Key, d.Slots.Value, d.Slots.Fn} + return append(spans, d.Slots.Args...) +} + +// filledSlots — the slot names this decision would fill. Text counts only when +// it differs from the whole utterance: fillSlots backfills the raw utterance +// into Text for a note, a query and a chat turn, so a set Text is not by itself +// evidence that anything was extracted. +func filledSlots(s Slots) []string { + var out []string + if s.HasTime { + out = append(out, "time") + } + if s.HasFn { + out = append(out, "fn") + } + if s.HasKey { + out = append(out, "key") + } + if s.Text != "" { + out = append(out, "text") + } + return out +} + +// bandOf — which kind of evidence this decision rests on. +// +// Stage 0 is anchored: a literal pattern matched and its span decided the +// intent. The LLM path (stage 1) is structural: the model read the whole +// sentence, and gateLLMDecision already checked the route for structural +// holes. The classifier (stages 2 and 3) is nearest, and the measurement is why +// it is one band rather than a scale — on the 91-case RU fixture its cosine +// spans 0.859 to 0.942 and scores 62% at both ends. +// +// A decision carrying a veto lands in BandVetoed regardless of who produced it. +// That is the point of the band: a self-vetoed claim should lose to any claim +// that is not, whatever machinery built it. +func bandOf(d Decision) claim.Band { + if vetoOf(d) != "" { + return claim.BandVetoed + } + switch d.Stage { + case 0: + return claim.BandAnchored + case 1: + return claim.BandStructural + default: + return claim.BandNearest + } +} + +// vetoOf — why this decision should not win, recovered as a reason rather than +// a number. +// +// gateLLMDecision flattens three named structural holes into +// llmThinConfidence, and the reason is lost at that point: 0.3 tells a reader +// that something was wrong and never which thing. The same three conditions are +// checked here so the claim carries the sentence a trace can print and the +// owner can be told. +// +// Clarify is checked last and is the general case. A decision below threshold +// has already asked to be doubted, whichever path set it. +func vetoOf(d Decision) string { + switch { + case d.Intent == IntentFact && !d.Slots.HasKey: + return "fact with no key: nothing to write, or a confident write under the wrong key" + case d.Intent == IntentAct && !d.Slots.HasFn: + return "act with no allowlisted fn: running an unlisted command or silently doing nothing" + case d.Intent == IntentReminder && !reminderHasSubject(d.Slots.Text): + return "reminder with no subject: it would fire empty at the hour" + case d.Clarify: + return "below the confidence gate" + } + return "" +} diff --git a/internal/router/claim_test.go b/internal/router/claim_test.go new file mode 100644 index 0000000..e9dd784 --- /dev/null +++ b/internal/router/claim_test.go @@ -0,0 +1,160 @@ +package router + +import ( + "context" + "reflect" + "testing" + "time" + + "github.com/kami/maven/internal/claim" +) + +func TestClaimOfBands(t *testing.T) { + cases := []struct { + name string + dec Decision + want claim.Band + }{ + { + name: "stage 0 is anchored", + dec: Decision{ + Utterance: "сколько времени", Stage: 0, Intent: IntentSystem, + Confidence: 1.0, Slots: Slots{Text: "сколько времени"}, + }, + want: claim.BandAnchored, + }, + { + name: "the llm path is structural", + dec: Decision{ + Utterance: "выпил воды", Stage: 1, Intent: IntentFact, + Confidence: llmFullConfidence, + Slots: Slots{Key: "water", Value: "1", HasKey: true, Text: "выпил воды"}, + }, + want: claim.BandStructural, + }, + { + name: "the classifier is nearest", + dec: Decision{ + Utterance: "что нового", Stage: 2, Intent: IntentQuery, + Confidence: 0.91, Slots: Slots{Text: "что нового"}, + }, + want: claim.BandNearest, + }, + { + name: "a structural hole vetoes whoever found it", + dec: Decision{ + Utterance: "запиши", Stage: 1, Intent: IntentFact, + Confidence: llmThinConfidence, Slots: Slots{Text: "запиши"}, + }, + want: claim.BandVetoed, + }, + { + name: "clarify vetoes a classifier decision", + dec: Decision{ + Utterance: "сделай это", Stage: 3, Intent: IntentNote, + Confidence: 0.2, Clarify: true, Slots: Slots{Text: "сделай это"}, + }, + want: claim.BandVetoed, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := ClaimOf("test", tc.dec) + if got.Band != tc.want { + t.Errorf("band = %v, want %v (veto %q)", got.Band, tc.want, got.Veto) + } + }) + } +} + +// The veto has to name the hole. Folding all three arms into llmThinConfidence +// is what lost the reason, and 0.3 tells a reader that something was wrong but +// never which thing. +func TestClaimOfVetoNamesTheHole(t *testing.T) { + cases := []struct { + name string + dec Decision + want string + }{ + {"keyless fact", Decision{Intent: IntentFact}, "fact with no key"}, + {"act with no fn", Decision{Intent: IntentAct}, "act with no allowlisted fn"}, + {"subjectless reminder", Decision{Intent: IntentReminder, Slots: Slots{Text: "напомни"}}, "reminder with no subject"}, + {"below the gate", Decision{Intent: IntentQuery, Clarify: true}, "below the confidence gate"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := ClaimOf("test", tc.dec) + if !got.Vetoed() { + t.Fatalf("no veto, want one about %q", tc.want) + } + if len(got.Veto) < len(tc.want) || got.Veto[:len(tc.want)] != tc.want { + t.Errorf("veto = %q, want it to start with %q", got.Veto, tc.want) + } + }) + } +} + +// A route with every slot filled must NOT be vetoed. The three arms are +// structural holes, not a tax on every decision. +func TestClaimOfCompleteRouteIsNotVetoed(t *testing.T) { + d := Decision{ + Utterance: "напомни позвонить маме в семь", Stage: 1, Intent: IntentReminder, + Confidence: llmFullConfidence, + Slots: Slots{Text: "позвонить маме", Time: time.Now(), HasTime: true}, + } + c := ClaimOf("llm", d) + if c.Vetoed() { + t.Errorf("complete reminder vetoed: %q", c.Veto) + } + if c.Band != claim.BandStructural { + t.Errorf("band = %v, want structural", c.Band) + } +} + +func TestClaimOfCoverageAndFilledSlots(t *testing.T) { + d := Decision{ + Utterance: "напомни позвонить маме", Stage: 0, Intent: IntentReminder, + Confidence: 1.0, Slots: Slots{Text: "позвонить маме"}, + } + c := ClaimOf("reminder-wakeword", d) + // The grammar captures what follows the verb, so "напомни" itself is + // unexplained. Under-reporting is the safe direction. + if len(c.Consumed) != 2 || len(c.Unexplained) != 1 { + t.Errorf("consumed %q / unexplained %q, want 2 and 1", c.Consumed, c.Unexplained) + } + if got := c.Coverage(); got < 0.66 || got > 0.67 { + t.Errorf("coverage = %v, want about 2/3", got) + } + if c.Intent != string(IntentReminder) { + t.Errorf("intent = %q", c.Intent) + } + if len(c.Filled) != 1 || c.Filled[0] != "text" { + t.Errorf("filled = %q, want [text]", c.Filled) + } + if c.Claimant != "reminder-wakeword" { + t.Errorf("claimant = %q", c.Claimant) + } +} + +// The point of V-565's "additive" constraint, asserted rather than trusted: +// building a claim reads a Decision and changes nothing about it, so the +// classifier floor and the two consumers of Confidence are untouched. +func TestClaimOfLeavesTheDecisionAlone(t *testing.T) { + r := New(Config{ + Grammars: []Grammar{ReminderGrammar()}, + Extractor: Extractor{}, + Threshold: 0.55, + }) + before, err := r.Route(context.Background(), "напомни полить цветы", time.Now()) + if err != nil { + t.Fatalf("Route: %v", err) + } + after := before + _ = ClaimOf("reminder-wakeword", after) + if !reflect.DeepEqual(after, before) { + t.Errorf("ClaimOf mutated the decision: %+v vs %+v", after, before) + } + if before.Confidence != 1.0 { + t.Errorf("stage 0 confidence = %v, want 1.0 — the float still has to work", before.Confidence) + } +}