build a claim from a Decision, beside the existing path (V-565)
router.ClaimOf maps a Decision onto the common unit. Stage 0 is anchored, the LLM path is structural, the classifier is nearest, and anything with a structural hole is vetoed whoever produced it. The veto recovers the reason gateLLMDecision throws away. Folding three named holes into llmThinConfidence leaves 0.3, which says something was wrong and never which thing, so the same conditions are read here as sentences a trace can print. Nothing in Route calls this. Decision.Confidence keeps its float and keeps working, because r.threshold and gateLLMDecision read it and the classifier is the failure floor. TestClaimOfLeavesTheDecisionAlone asserts that. TestONNXBaseline is unchanged at 64/91.
This commit is contained in:
@@ -0,0 +1,113 @@
|
||||
package router
|
||||
|
||||
import "github.com/kami/maven/internal/claim"
|
||||
|
||||
// ClaimOf — build a claim.Claim from a Decision (V-565, design in
|
||||
// docs/plans/19-dialogue-arbitration.md).
|
||||
//
|
||||
// Additive and beside the existing path. Decision.Confidence keeps its float
|
||||
// and keeps working: r.threshold and gateLLMDecision read it, and the
|
||||
// classifier cascade is the failure floor. Nothing in Route calls this yet.
|
||||
// The arbiter that reads claims is V-560.
|
||||
//
|
||||
// claimant names who produced the decision. The cascade does not record which
|
||||
// stage-0 grammar matched, so the caller passes what it knows and the builder
|
||||
// does not guess.
|
||||
func ClaimOf(claimant string, d Decision) claim.Claim {
|
||||
consumed, unexplained := claim.Split(d.Utterance, claimSpans(d)...)
|
||||
return claim.Claim{
|
||||
Claimant: claimant,
|
||||
Intent: string(d.Intent),
|
||||
Filled: filledSlots(d.Slots),
|
||||
Consumed: consumed,
|
||||
Unexplained: unexplained,
|
||||
Band: bandOf(d),
|
||||
Veto: vetoOf(d),
|
||||
}
|
||||
}
|
||||
|
||||
// claimSpans — the parts of the utterance the decision says it read. Slot
|
||||
// values, not the utterance, because coverage is the question of how much of
|
||||
// the sentence the claim actually explains.
|
||||
//
|
||||
// A stage-0 grammar reports whatever its Build put in the slots, which for the
|
||||
// reminder rule is the text after "напомни" and not the verb itself. That
|
||||
// under-reports coverage rather than over-reporting it, which is the safe
|
||||
// direction: a claim that overstates what it explains wins arbitrations it
|
||||
// should lose.
|
||||
func claimSpans(d Decision) []string {
|
||||
spans := []string{d.Slots.Text, d.Slots.Key, d.Slots.Value, d.Slots.Fn}
|
||||
return append(spans, d.Slots.Args...)
|
||||
}
|
||||
|
||||
// filledSlots — the slot names this decision would fill. Text counts only when
|
||||
// it differs from the whole utterance: fillSlots backfills the raw utterance
|
||||
// into Text for a note, a query and a chat turn, so a set Text is not by itself
|
||||
// evidence that anything was extracted.
|
||||
func filledSlots(s Slots) []string {
|
||||
var out []string
|
||||
if s.HasTime {
|
||||
out = append(out, "time")
|
||||
}
|
||||
if s.HasFn {
|
||||
out = append(out, "fn")
|
||||
}
|
||||
if s.HasKey {
|
||||
out = append(out, "key")
|
||||
}
|
||||
if s.Text != "" {
|
||||
out = append(out, "text")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// bandOf — which kind of evidence this decision rests on.
|
||||
//
|
||||
// Stage 0 is anchored: a literal pattern matched and its span decided the
|
||||
// intent. The LLM path (stage 1) is structural: the model read the whole
|
||||
// sentence, and gateLLMDecision already checked the route for structural
|
||||
// holes. The classifier (stages 2 and 3) is nearest, and the measurement is why
|
||||
// it is one band rather than a scale — on the 91-case RU fixture its cosine
|
||||
// spans 0.859 to 0.942 and scores 62% at both ends.
|
||||
//
|
||||
// A decision carrying a veto lands in BandVetoed regardless of who produced it.
|
||||
// That is the point of the band: a self-vetoed claim should lose to any claim
|
||||
// that is not, whatever machinery built it.
|
||||
func bandOf(d Decision) claim.Band {
|
||||
if vetoOf(d) != "" {
|
||||
return claim.BandVetoed
|
||||
}
|
||||
switch d.Stage {
|
||||
case 0:
|
||||
return claim.BandAnchored
|
||||
case 1:
|
||||
return claim.BandStructural
|
||||
default:
|
||||
return claim.BandNearest
|
||||
}
|
||||
}
|
||||
|
||||
// vetoOf — why this decision should not win, recovered as a reason rather than
|
||||
// a number.
|
||||
//
|
||||
// gateLLMDecision flattens three named structural holes into
|
||||
// llmThinConfidence, and the reason is lost at that point: 0.3 tells a reader
|
||||
// that something was wrong and never which thing. The same three conditions are
|
||||
// checked here so the claim carries the sentence a trace can print and the
|
||||
// owner can be told.
|
||||
//
|
||||
// Clarify is checked last and is the general case. A decision below threshold
|
||||
// has already asked to be doubted, whichever path set it.
|
||||
func vetoOf(d Decision) string {
|
||||
switch {
|
||||
case d.Intent == IntentFact && !d.Slots.HasKey:
|
||||
return "fact with no key: nothing to write, or a confident write under the wrong key"
|
||||
case d.Intent == IntentAct && !d.Slots.HasFn:
|
||||
return "act with no allowlisted fn: running an unlisted command or silently doing nothing"
|
||||
case d.Intent == IntentReminder && !reminderHasSubject(d.Slots.Text):
|
||||
return "reminder with no subject: it would fire empty at the hour"
|
||||
case d.Clarify:
|
||||
return "below the confidence gate"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/claim"
|
||||
)
|
||||
|
||||
func TestClaimOfBands(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
dec Decision
|
||||
want claim.Band
|
||||
}{
|
||||
{
|
||||
name: "stage 0 is anchored",
|
||||
dec: Decision{
|
||||
Utterance: "сколько времени", Stage: 0, Intent: IntentSystem,
|
||||
Confidence: 1.0, Slots: Slots{Text: "сколько времени"},
|
||||
},
|
||||
want: claim.BandAnchored,
|
||||
},
|
||||
{
|
||||
name: "the llm path is structural",
|
||||
dec: Decision{
|
||||
Utterance: "выпил воды", Stage: 1, Intent: IntentFact,
|
||||
Confidence: llmFullConfidence,
|
||||
Slots: Slots{Key: "water", Value: "1", HasKey: true, Text: "выпил воды"},
|
||||
},
|
||||
want: claim.BandStructural,
|
||||
},
|
||||
{
|
||||
name: "the classifier is nearest",
|
||||
dec: Decision{
|
||||
Utterance: "что нового", Stage: 2, Intent: IntentQuery,
|
||||
Confidence: 0.91, Slots: Slots{Text: "что нового"},
|
||||
},
|
||||
want: claim.BandNearest,
|
||||
},
|
||||
{
|
||||
name: "a structural hole vetoes whoever found it",
|
||||
dec: Decision{
|
||||
Utterance: "запиши", Stage: 1, Intent: IntentFact,
|
||||
Confidence: llmThinConfidence, Slots: Slots{Text: "запиши"},
|
||||
},
|
||||
want: claim.BandVetoed,
|
||||
},
|
||||
{
|
||||
name: "clarify vetoes a classifier decision",
|
||||
dec: Decision{
|
||||
Utterance: "сделай это", Stage: 3, Intent: IntentNote,
|
||||
Confidence: 0.2, Clarify: true, Slots: Slots{Text: "сделай это"},
|
||||
},
|
||||
want: claim.BandVetoed,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := ClaimOf("test", tc.dec)
|
||||
if got.Band != tc.want {
|
||||
t.Errorf("band = %v, want %v (veto %q)", got.Band, tc.want, got.Veto)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The veto has to name the hole. Folding all three arms into llmThinConfidence
|
||||
// is what lost the reason, and 0.3 tells a reader that something was wrong but
|
||||
// never which thing.
|
||||
func TestClaimOfVetoNamesTheHole(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
dec Decision
|
||||
want string
|
||||
}{
|
||||
{"keyless fact", Decision{Intent: IntentFact}, "fact with no key"},
|
||||
{"act with no fn", Decision{Intent: IntentAct}, "act with no allowlisted fn"},
|
||||
{"subjectless reminder", Decision{Intent: IntentReminder, Slots: Slots{Text: "напомни"}}, "reminder with no subject"},
|
||||
{"below the gate", Decision{Intent: IntentQuery, Clarify: true}, "below the confidence gate"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := ClaimOf("test", tc.dec)
|
||||
if !got.Vetoed() {
|
||||
t.Fatalf("no veto, want one about %q", tc.want)
|
||||
}
|
||||
if len(got.Veto) < len(tc.want) || got.Veto[:len(tc.want)] != tc.want {
|
||||
t.Errorf("veto = %q, want it to start with %q", got.Veto, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A route with every slot filled must NOT be vetoed. The three arms are
|
||||
// structural holes, not a tax on every decision.
|
||||
func TestClaimOfCompleteRouteIsNotVetoed(t *testing.T) {
|
||||
d := Decision{
|
||||
Utterance: "напомни позвонить маме в семь", Stage: 1, Intent: IntentReminder,
|
||||
Confidence: llmFullConfidence,
|
||||
Slots: Slots{Text: "позвонить маме", Time: time.Now(), HasTime: true},
|
||||
}
|
||||
c := ClaimOf("llm", d)
|
||||
if c.Vetoed() {
|
||||
t.Errorf("complete reminder vetoed: %q", c.Veto)
|
||||
}
|
||||
if c.Band != claim.BandStructural {
|
||||
t.Errorf("band = %v, want structural", c.Band)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaimOfCoverageAndFilledSlots(t *testing.T) {
|
||||
d := Decision{
|
||||
Utterance: "напомни позвонить маме", Stage: 0, Intent: IntentReminder,
|
||||
Confidence: 1.0, Slots: Slots{Text: "позвонить маме"},
|
||||
}
|
||||
c := ClaimOf("reminder-wakeword", d)
|
||||
// The grammar captures what follows the verb, so "напомни" itself is
|
||||
// unexplained. Under-reporting is the safe direction.
|
||||
if len(c.Consumed) != 2 || len(c.Unexplained) != 1 {
|
||||
t.Errorf("consumed %q / unexplained %q, want 2 and 1", c.Consumed, c.Unexplained)
|
||||
}
|
||||
if got := c.Coverage(); got < 0.66 || got > 0.67 {
|
||||
t.Errorf("coverage = %v, want about 2/3", got)
|
||||
}
|
||||
if c.Intent != string(IntentReminder) {
|
||||
t.Errorf("intent = %q", c.Intent)
|
||||
}
|
||||
if len(c.Filled) != 1 || c.Filled[0] != "text" {
|
||||
t.Errorf("filled = %q, want [text]", c.Filled)
|
||||
}
|
||||
if c.Claimant != "reminder-wakeword" {
|
||||
t.Errorf("claimant = %q", c.Claimant)
|
||||
}
|
||||
}
|
||||
|
||||
// The point of V-565's "additive" constraint, asserted rather than trusted:
|
||||
// building a claim reads a Decision and changes nothing about it, so the
|
||||
// classifier floor and the two consumers of Confidence are untouched.
|
||||
func TestClaimOfLeavesTheDecisionAlone(t *testing.T) {
|
||||
r := New(Config{
|
||||
Grammars: []Grammar{ReminderGrammar()},
|
||||
Extractor: Extractor{},
|
||||
Threshold: 0.55,
|
||||
})
|
||||
before, err := r.Route(context.Background(), "напомни полить цветы", time.Now())
|
||||
if err != nil {
|
||||
t.Fatalf("Route: %v", err)
|
||||
}
|
||||
after := before
|
||||
_ = ClaimOf("reminder-wakeword", after)
|
||||
if !reflect.DeepEqual(after, before) {
|
||||
t.Errorf("ClaimOf mutated the decision: %+v vs %+v", after, before)
|
||||
}
|
||||
if before.Confidence != 1.0 {
|
||||
t.Errorf("stage 0 confidence = %v, want 1.0 — the float still has to work", before.Confidence)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user