From da2d11dab60855ca6493e527cc340f13b6d3426b Mon Sep 17 00:00:00 2001 From: claude Date: Thu, 6 Aug 2026 00:46:50 +0400 Subject: [PATCH 1/5] plan: the claim unit and its four bands, measured (V-565) Measures what each claimant on an utterance reports across the 91-case RU fixture, then argues an ordinal band set from that distribution. The classifier's cosine is flat against correctness: 62% correct below its median and 62% above, over a spread only 0.083 wide, with every case above the 0.55 gate. Its top-two margin is p50 0.009 and never reaches 0.03. So a calibrated float is not cheaply available and the ledger's assumption holds. Stage 0 is 20/20 on the cases it claims and asserts 1.0 for all of them. The LLM router emits two values, and the lower one is a self-veto with a reason flattened into a number. --- docs/plans/19-dialogue-arbitration.md | 135 ++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 docs/plans/19-dialogue-arbitration.md diff --git a/docs/plans/19-dialogue-arbitration.md b/docs/plans/19-dialogue-arbitration.md new file mode 100644 index 0000000..0d5c36e --- /dev/null +++ b/docs/plans/19-dialogue-arbitration.md @@ -0,0 +1,135 @@ +# Plan: dialogue arbitration, one channel and many claimants + +Umbrella V-558. This file collects the design for its children. + +Last verified: 06-08-2026 @ b6305f1 + +## A common unit for claims on an utterance (V-565) + +**Verdict: four ordinal bands, and the band is the tie-break rather than the decision. +Coverage decides first.** The measurement below says no claimant Maven has today can produce +a graded confidence. A float would be an invention either way. What is available is the KIND +of evidence a claimant holds, and there are exactly four kinds. + +### What the claimants report today + +Measured 06-08-2026 on the 91-case RU fixture (`internal/router/eval`), through the deployed +cascade with the quantized multilingual-e5-small embedder. The harness is +`TestONNXClaimConfidenceDistribution` and `TestStage0Contention` in +`internal/router/eval/claims_test.go`. Correct means the right intent, or a refusal where the +fixture wants one. Slots are excluded, because a slot miss is a parser question and would +blur what the number is being asked to predict. + +| Claimant | Values it can emit | Distribution on the fixture | Correct | +|---|---|---|---| +| Stage 0 grammars, 21 of them | `1.0`, always | claimed 20 of 91 cases | 20/20 (100%) | +| Classifier, cosine | continuous in principle | observed range 0.859 to 0.942 over 71 cases | 44/71 (62%) | +| LLM router | `1.0` or `0.3`, nothing between | not run here, no llama-server | see below | +| Query sources, 22 of them | a bool | not routed by the fixture | n/a | +| Stateful four | nothing at all | n/a | n/a | + +Four findings, and each one constrains the band set. + +**The classifier's cosine carries no signal about correctness.** It scores 62% below the +median and 62% above it. That is 13/21 in 0.8 to 0.9, and 31/50 in 0.9 to 1.0. The spread is +0.083 wide. Every case sits above the 0.55 threshold, so the gate never fires here. A number +flat against correctness, which never crosses its own gate, is not a confidence. + +**Nor does the margin between its top two intents.** Top1 minus top2 is min 0.000, p50 +0.009, max 0.025. Sixty-eight of the 71 classified cases sit under 0.02 and score 60%. Three +clear 0.02 and score 3/3, which is a sample of three. So the ledger's question is answered: +a calibrated float is NOT cheaply available from the classifier alone. Nearest-centroid over +frozen seeds ranks intents, and the ranking is decided in the third decimal place. It can say +which intent is nearest. It cannot say how near. + +**Stage 0 asserts 1.0 by fiat, and on this fixture the fiat is right.** Twenty of twenty. +That is not evidence that a hand-written anchored pattern is always right. It is evidence +that anchored and nearest are different kinds of claim, and must not share a scale. The gap +is 100% against 62% on the same 91 utterances. + +**Stage 0 contention is rarer than the list order suggests.** Exactly one case of 91 draws +two grammars. That is `ru-query-019`, where `calendar-query` and `agenda-query` both match, +and `calendar-query` wins because it is earlier in `buildRouter`. Both would route +`IntentQuery`, so the ordering costs nothing there. The finding is not that ordering is +harmless. It is that the fixture barely exercises what V-558 is about. Part of what a claim +object buys is making the contention countable. + +**The LLM router emits two values, and one of them is not a confidence.** `llmFullConfidence` +is 1.0 and `llmThinConfidence` is 0.3. `gateLLMDecision` moves a decision to 0.3 through +three named arms. A fact with no key, an act with no allowlisted fn, a reminder with no +subject. Each is a self-veto with a reason, flattened into a number that then loses the +reason. Both values are meaningful only against `config.DefaultRouterThreshold`. 0.3 is below +0.55 and 1.0 is above it, and nothing anywhere reads any other property of either. + +### The band set + +Four bands, ordinal, highest first. They name the kind of evidence, because that is the one +thing every claimant can report without inventing it. + +**`BandAnchored`.** A literal pattern anchored in the utterance matched, and the matched span +is what decides the intent. Stage 0 grammars and query-source matchers. The claimant is +certain about the shape of the sentence. That is not the same as being certain about the +answer. Measured 20/20. + +**`BandStructural`.** A claimant read the whole sentence and produced a complete route. Every +slot the intent requires is filled. The LLM router at `llmFullConfidence` sits here, and so +does a stateful claimant holding a pending question. Not anchored, because nothing in the +utterance is pointed at. + +**`BandNearest`.** The claim rests only on resemblance to something else. No anchor in the +utterance, no structural check behind it. The classifier. One band rather than a graded +scale, and the measurement is the argument. 62% at both ends of the cosine range, and a +top-two margin that never reaches 0.03. + +**`BandVetoed`.** The claimant will take the turn only if nobody else will, and says why it +should not. The three arms of `gateLLMDecision` land here with their reason preserved. A +vetoed claim is still a claim. Maven asking "о чём напомнить?" beats silence. + +There is no fifth band, and that is a measurement result rather than a preference. No +claimant in the cascade today can report what a fifth band would carry. V-546 lands a softmax +head whose max probability is a calibrated number. That one gets read as a number, not +squeezed into these four. + +### Coverage decides before the band does + +The band is the tie-break. The first question is how much of the utterance a claim explains, +and that is `Consumed` against `Unexplained` on the claim object. Two reasons. + +It is the fix for the failure that opened V-558. "какая сейчас погода в Риме?" arrived while +a reminder was pending. The pending claimant ate the whole utterance as a time answer while +explaining none of it. Not "погода", not "Риме", not the question mark. A weather claim +explains all of it. Coverage-first arbitration prefers the weather claim without knowing that +a pending reminder is less trustworthy than a grammar. The pending question then survives to +be asked again. + +It also keeps the stateful four out of the top slot without special-casing them. They sit at +`BandStructural`, below any anchored claim. That is the whole V-558 complaint about the +highest-priority claimants being the least informed, expressed as one rule. + +### The claim object + +```go +type Claim struct { + Claimant string // who wants the turn + Intent string // plain string: internal/dialogue must not import internal/router + Filled []string // the slots this claim would fill + Consumed []string // utterance tokens this claim explains + Unexplained []string // the rest, in order + Band Band + Veto string // why this claim should NOT win, empty when there is none +} +``` + +`Intent` is a plain `string` rather than `router.Intent` on purpose. `internal/dialogue` must +not import `internal/router`, so the claim package must not either, and a shared string costs +one conversion at each edge. + +`Unexplained` is carried rather than derived at read time. A claimant can then decline to +explain a span it did match. + +### What this task does not do + +`router.Decision.Confidence` stays and keeps its float. `r.threshold` and `gateLLMDecision` +read it, and the classifier is the failure floor. A rewire that broke either would trade a +measured floor for an unmeasured design. V-565 lands the type and the builder beside the +existing path. The arbiter that reads claims is V-560. From 934a28d67c202e5e9768836722a9501cb1eddcd1 Mon Sep 17 00:00:00 2001 From: claude Date: Thu, 6 Aug 2026 00:54:13 +0400 Subject: [PATCH 2/5] measure what each claimant on an utterance reports (V-565) Two reporting tests over the 91-case RU fixture, no ratchet: a ratchet here would freeze a number nobody has decided to hold. TestStage0Contention runs the 21 grammars one at a time instead of stopping at the first match. One case of 91 draws two, ru-query-019, where calendar-query beats agenda-query by list position alone. TestONNXClaimConfidenceDistribution buckets the reported confidence by the layer that produced it. Stage 0 is 20/20 at a hardcoded 1.0. The classifier scores 62% below its median and 62% above, across a cosine range of 0.859 to 0.942, with a top-two margin of p50 0.009. The float is not a confidence. newBaselineClassifier and baselineGrammars split out of newBaselineRouter so the measurement runs the same rules the daemon runs. TestONNXBaseline is unchanged at 64/91. --- internal/router/eval/claims_test.go | 230 ++++++++++++++++++++++++++++ internal/router/eval/eval_test.go | 44 ++++-- 2 files changed, 261 insertions(+), 13 deletions(-) create mode 100644 internal/router/eval/claims_test.go diff --git a/internal/router/eval/claims_test.go b/internal/router/eval/claims_test.go new file mode 100644 index 0000000..29f53ad --- /dev/null +++ b/internal/router/eval/claims_test.go @@ -0,0 +1,230 @@ +package eval + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "testing" + + "github.com/kami/maven/internal/router" +) + +// Package-level note for V-565. The cascade's arbitration is list order, and +// the reason is that no two claimants report a comparable number. These tests +// measure what each claimant actually reports across the 91-case RU fixture, +// so the ordinal band set in docs/plans/19-dialogue-arbitration.md is argued +// from a distribution rather than from taste. They report and never assert: +// a ratchet here would freeze a number nobody has decided to hold yet. + +// TestStage0Contention — how often more than one stage-0 grammar matches the +// same utterance. Every one of them reports Confidence 1.0, so where two +// match, list order is the entire decision and nothing in the Decision says a +// second rule wanted the turn. +func TestStage0Contention(t *testing.T) { + f, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + grammars := baselineGrammars(router.DefaultActMatcher{Fns: actFns}) + t.Logf("stage 0: %d grammars over %d cases", len(grammars), len(f.Cases)) + + matched, contended := 0, 0 + pairs := map[string]int{} + for _, c := range f.Cases { + claimants := matchingGrammars(grammars, c.Utterance) + if len(claimants) == 0 { + continue + } + matched++ + if len(claimants) < 2 { + continue + } + contended++ + t.Logf(" contended %s %q: %v (winner %q by order)", c.ID, c.Utterance, claimants, claimants[0]) + for _, loser := range claimants[1:] { + pairs[claimants[0]+" beats "+loser]++ + } + } + t.Logf("stage 0 claimed %d/%d cases, %d of those with more than one claimant", matched, len(f.Cases), contended) + for _, k := range sortedKeys(pairs) { + t.Logf(" %s ×%d", k, pairs[k]) + } +} + +// matchingGrammars — every grammar whose pattern matches AND whose Build +// accepts, in the daemon's order. Route stops at the first; this does not. +func matchingGrammars(grammars []router.Grammar, utterance string) []string { + stripped, hadWake := router.StripWakeToken(utterance) + var out []string + for _, g := range grammars { + m := g.Pattern.FindStringSubmatch(utterance) + if m == nil && hadWake { + m = g.Pattern.FindStringSubmatch(stripped) + } + if m == nil { + continue + } + if _, ok := g.Build(m); !ok { + continue + } + out = append(out, g.Name) + } + return out +} + +// TestClaimConfidenceDistributionHash — the confidence each claimant reports, +// on the deterministic hash embedder so it runs anywhere. The ONNX run below +// is the one whose cosines are the deployed numbers. +func TestClaimConfidenceDistributionHash(t *testing.T) { + reportConfidences(t, "hash", router.NewHashEmbedder(1024)) +} + +// TestONNXClaimConfidenceDistribution — the same measurement on the embedder +// homesrv runs, so the cosine column is the real one. Opt-in via +// MAVEN_ONNX_LIB, same as TestONNXBaseline, and one TestONNX* per process. +func TestONNXClaimConfidenceDistribution(t *testing.T) { + lib := os.Getenv("MAVEN_ONNX_LIB") + if lib == "" { + t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing") + } + model := filepath.Join("../../..", "models/embedder/multilingual-e5-small/model_quantized.onnx") + tok := filepath.Join("../../..", "models/embedder/multilingual-e5-small/tokenizer.json") + for _, p := range []string{lib, model, tok} { + if _, err := os.Stat(p); err != nil { + t.Skipf("missing %s: %v", p, err) + } + } + emb, err := router.NewONNXEmbedder(model, tok, lib) + if err != nil { + t.Skipf("onnx embedder unavailable: %v", err) + } + defer emb.Close() + reportConfidences(t, "onnx", emb) +} + +// reportConfidences runs the fixture through the deployed cascade and buckets +// the reported confidence by which layer produced it, then reports how well +// each bucket predicts a correct route. A band is only worth defining if the +// accuracy inside it differs from the accuracy outside it. +func reportConfidences(t *testing.T, name string, emb router.Embedder) { + t.Helper() + f, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + now, err := f.Now() + if err != nil { + t.Fatalf("Now: %v", err) + } + r := newBaselineRouter(t, emb, nil) + cls := newBaselineClassifier(t, emb) + + type bucket struct{ n, correct int } + byValue := map[string]*bucket{} + byMargin := map[string]*bucket{} + var cosines, margins []float64 + for _, c := range f.Cases { + d, err := r.Route(context.Background(), c.Utterance, now) + if err != nil { + t.Fatalf("%s: %v", c.ID, err) + } + layer := "classifier" + if d.Stage == 0 { + layer = "stage0" + } else { + cosines = append(cosines, d.Confidence) + } + key := fmt.Sprintf("%s conf=%.2f", layer, d.Confidence) + if layer == "classifier" { + key = fmt.Sprintf("%s conf=%.1f..%.1f", layer, floorTo(d.Confidence, 0.1), floorTo(d.Confidence, 0.1)+0.1) + } + b := byValue[key] + if b == nil { + b = &bucket{} + byValue[key] = b + } + ok := routeCorrect(c, d) + b.n++ + if ok { + b.correct++ + } + + // The margin between the classifier's top two intents is the other + // float one could call a confidence. Measured on the same cases, so + // the ledger's "is a calibrated float available cheaply" question + // gets an answer instead of an assumption. + if layer != "classifier" { + continue + } + res, err := cls.Classify(context.Background(), c.Utterance) + if err != nil || len(res) < 2 { + continue + } + margin := res[0].Score - res[1].Score + margins = append(margins, margin) + mk := fmt.Sprintf("margin %.2f..%.2f", floorTo(margin, 0.02), floorTo(margin, 0.02)+0.02) + mb := byMargin[mk] + if mb == nil { + mb = &bucket{} + byMargin[mk] = mb + } + mb.n++ + if ok { + mb.correct++ + } + } + + t.Logf("%s: confidence buckets over %d cases (correct = right intent, or clarified when the fixture wants a refusal)", name, len(f.Cases)) + for _, k := range sortedKeys2(byValue) { + b := byValue[k] + t.Logf(" %-32s n=%2d correct=%2d (%.0f%%)", k, b.n, b.correct, 100*float64(b.correct)/float64(b.n)) + } + if len(cosines) > 0 { + sort.Float64s(cosines) + t.Logf(" classifier cosine spread: min %.3f p25 %.3f p50 %.3f p75 %.3f max %.3f", + cosines[0], cosines[len(cosines)/4], cosines[len(cosines)/2], + cosines[3*len(cosines)/4], cosines[len(cosines)-1]) + } + if len(margins) > 0 { + sort.Float64s(margins) + t.Logf(" classifier top1-top2 margin: min %.3f p50 %.3f max %.3f", + margins[0], margins[len(margins)/2], margins[len(margins)-1]) + for _, k := range sortedKeys2(byMargin) { + b := byMargin[k] + t.Logf(" %-32s n=%2d correct=%2d (%.0f%%)", k, b.n, b.correct, 100*float64(b.correct)/float64(b.n)) + } + } +} + +// routeCorrect — the intent contract only. Slots are a parser question and +// would blur what the confidence number is being asked to predict. +func routeCorrect(c Case, d router.Decision) bool { + if c.WantClarify { + return d.Clarify + } + return d.Intent == c.Intent && !d.Clarify +} + +func floorTo(v, step float64) float64 { + return float64(int(v/step)) * step +} + +func sortedKeys(m map[string]int) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func sortedKeys2[T any](m map[string]T) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/router/eval/eval_test.go b/internal/router/eval/eval_test.go index b3ea537..8b8ee8b 100644 --- a/internal/router/eval/eval_test.go +++ b/internal/router/eval/eval_test.go @@ -216,6 +216,27 @@ func TestONNXBaseline(t *testing.T) { func newBaselineRouter(t *testing.T, emb router.Embedder, llmR *router.LLMRouter) *router.Router { t.Helper() acts := router.DefaultActMatcher{Fns: actFns} + cls := newBaselineClassifier(t, emb) + return router.New(router.Config{ + Grammars: baselineGrammars(acts), + Classifier: cls, + Extractor: router.Extractor{ + Time: router.StubDateTimeParser{}, + Acts: acts, + Facts: router.DefaultFactParser{}, + }, + // The deployed gate, not a test-local one: a fixture scored at a looser + // threshold reports an accuracy no real turn would see. + Threshold: config.DefaultRouterThreshold, + LLM: llmR, + }) +} + +// newBaselineClassifier — the seeded nearest-centroid classifier the cascade +// runs. Split out of newBaselineRouter so the claim measurement can ask it for +// its full ranking, not just the winner the Decision carries. +func newBaselineClassifier(t *testing.T, emb router.Embedder) *router.Classifier { + t.Helper() cls := router.NewClassifier(emb) ctx := context.Background() seeds := seedsWithIntent(t) @@ -231,6 +252,15 @@ func newBaselineRouter(t *testing.T, emb router.Embedder, llmR *router.LLMRouter t.Fatalf("seed %q: %v", text, err) } } + return cls +} + +// baselineGrammars — the stage-0 rule set in the daemon's order (buildRouter in +// cmd/mavend/voicewire.go). Split out of newBaselineRouter so the claim +// measurement can run the same rules one at a time and see which of them +// contend for the same utterance, which the cascade hides by stopping at the +// first match. +func baselineGrammars(acts router.ActMatcher) []router.Grammar { grammars := router.DefaultGrammars(acts) grammars = append(grammars, router.SystemTimeDateGrammars()...) // Same order as buildRouter (voicewire.go). The fixture is only worth @@ -248,19 +278,7 @@ func newBaselineRouter(t *testing.T, emb router.Embedder, llmR *router.LLMRouter // "расскажи про X" is a world question the model called a fact, and the // rule goes last because it matches on the first word alone (Vikunja #498). grammars = append(grammars, router.NarrativeQueryGrammars()...) - return router.New(router.Config{ - Grammars: grammars, - Classifier: cls, - Extractor: router.Extractor{ - Time: router.StubDateTimeParser{}, - Acts: acts, - Facts: router.DefaultFactParser{}, - }, - // The deployed gate, not a test-local one: a fixture scored at a looser - // threshold reports an accuracy no real turn would see. - Threshold: config.DefaultRouterThreshold, - LLM: llmR, - }) + return grammars } // seedOrder — fixed iteration order over the corpus. Not cosmetic: a few From e031f8f5f5c47f78ecbcd3c64725a15bec6c09c4 Mon Sep 17 00:00:00 2001 From: claude Date: Thu, 6 Aug 2026 00:54:34 +0400 Subject: [PATCH 3/5] a claim carries evidence, not a verdict (V-565) internal/claim holds the common unit: who wants the turn, the intent, the slots it would fill, the tokens it explains, the tokens it declines, and why it should not win. Specificity reads Consumed against Unexplained and negative constraint reads Veto, so neither needs a float. Four ordinal bands where a number is unavoidable, argued from measurement in docs/plans/19-dialogue-arbitration.md. Anchored, structural, nearest, vetoed. Nearest is one band and not a scale because the classifier's cosine measured flat against correctness. MoreSpecificThan puts coverage first and the band second. That is the fix for the failure V-558 opened with: a pending reminder ate the Rome weather question while explaining none of it. The package imports nothing from the rest of Maven. internal/dialogue must not import internal/router, so Intent is a plain string. --- internal/claim/claim.go | 192 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 internal/claim/claim.go diff --git a/internal/claim/claim.go b/internal/claim/claim.go new file mode 100644 index 0000000..c704b4f --- /dev/null +++ b/internal/claim/claim.go @@ -0,0 +1,192 @@ +// Package claim is the common unit for the many claimants that compete for one +// utterance (V-565, umbrella V-558, design in +// docs/plans/19-dialogue-arbitration.md). +// +// Maven's cascade has roughly ten stage-0 grammars, seven router intents, +// twenty-two query sources and four stateful pre-emptors, and every one of them +// answers "is this mine?" alone. None can answer "is this more mine than +// yours?", because their scores are not comparable: stage 0 asserts 1.0 by +// fiat, the classifier reports a cosine, the LLM router derives one from +// structure. So list order is the whole arbitration. +// +// A Claim carries evidence rather than a verdict. Two things read that evidence +// and neither needs a float: +// +// - specificity — a claim explaining more of the utterance is preferred, and +// that is Consumed against Unexplained; +// - negative constraint — a claimant may veto itself and say why, and that is +// Veto. +// +// Where a number is unavoidable, it is an ordinal Band and not a probability. +// The band set is argued from measurement in the plan doc: the classifier's +// cosine is flat against correctness (62% at both ends of a spread 0.083 wide) +// and its top-two margin is p50 0.009, so no claimant Maven has today can +// produce a graded confidence. +// +// This package deliberately imports nothing from the rest of Maven. +// internal/dialogue must not import internal/router, so a shared unit that +// pulled in router.Intent would smuggle that edge back in. Intent is a plain +// string and the conversion happens at each edge. +package claim + +import "strings" + +// Band — the kind of evidence behind a claim, ordinal and comparable. Higher +// wins a tie. Four values, because four is what the claimants can report. +type Band int + +const ( + // BandUnknown — the zero value. A claim that never set a band is a bug in + // its builder, not a weak claim, so it must not silently rank as one. + BandUnknown Band = iota + + // BandVetoed — the claimant will take the turn only if nobody else will, + // and Veto says why it should not. The three arms of gateLLMDecision (a + // fact with no key, an act with no allowlisted fn, a reminder with no + // subject) land here. Still a claim: asking "о чём напомнить?" beats + // silence. + BandVetoed + + // BandNearest — the claim rests only on resemblance to something else, + // with no anchor in the utterance and no structural check behind it. The + // nearest-centroid classifier. One band and not a graded scale, because + // the cosine measured flat against correctness. + BandNearest + + // BandStructural — the claimant read the whole sentence and produced a + // complete route, every slot its intent requires filled. The LLM router at + // full confidence, and a stateful claimant holding a pending question. + // Below BandAnchored on purpose: the four stateful claimants pre-empt + // unconditionally today, and that is the V-558 defect. + BandStructural + + // BandAnchored — a literal pattern anchored in the utterance matched, and + // the matched span is what decides the intent. Stage 0 grammars and + // query-source matchers. Certainty about the shape of the sentence, which + // is not certainty about the answer. + BandAnchored +) + +// String — the band's name, for a trace line and for a test failure that has to +// say which band it got. +func (b Band) String() string { + switch b { + case BandVetoed: + return "vetoed" + case BandNearest: + return "nearest" + case BandStructural: + return "structural" + case BandAnchored: + return "anchored" + default: + return "unknown" + } +} + +// Claim — one claimant's bid for one utterance. +type Claim struct { + // Claimant — who wants the turn. A grammar name, a query source name, a + // stage label. Read by the trace and by a test naming a loser. + Claimant string + + // Intent — the route this claim would take. A plain string and not + // router.Intent: see the package comment. + Intent string + + // Filled — the slot names this claim would fill ("time", "fn", "key", + // "text"). Names and not values, because arbitration compares shape. + Filled []string + + // Consumed — the utterance tokens this claim explains, in the order they + // appear. The numerator of specificity. + Consumed []string + + // Unexplained — the tokens this claim does not explain, in order. Carried + // rather than derived, so a claimant may decline a span it did match. + Unexplained []string + + // Band — the kind of evidence. The tie-break, after coverage. + Band Band + + // Veto — why this claim should NOT win, empty when there is none. A + // non-empty Veto and a Band above BandVetoed is legal: a claim can be + // well-evidenced and still name a reason to prefer somebody else. + Veto string +} + +// Coverage — the fraction of the utterance this claim explains, in [0,1]. A +// claim with no tokens either way covers nothing; it is not division by zero +// and it is not a full claim. +func (c Claim) Coverage() float64 { + total := len(c.Consumed) + len(c.Unexplained) + if total == 0 { + return 0 + } + return float64(len(c.Consumed)) / float64(total) +} + +// Vetoed reports whether the claimant named a reason against itself. +func (c Claim) Vetoed() bool { return c.Veto != "" } + +// MoreSpecificThan — the ordering V-560's arbiter will read. Coverage first, +// because that is what fixes the failure this program opened with: a pending +// reminder ate "какая сейчас погода в Риме?" as a time answer while explaining +// none of it. Band only breaks a coverage tie. +// +// Deliberately NOT wired into the cascade by V-565. It is here so the ordering +// is one function with tests on it, rather than a rule restated at each of the +// sites that will eventually call it. +func (c Claim) MoreSpecificThan(other Claim) bool { + cc, oc := c.Coverage(), other.Coverage() + if cc != oc { + return cc > oc + } + return c.Band > other.Band +} + +// Tokens — the utterance split for coverage accounting. Whitespace, then +// trailing and leading punctuation, then lowercased. +// +// This is tokenization over the raw string and not a Russian pattern: it +// contains no word list, and its output is a count rather than a fact or a +// route (CLAUDE.md § Russian patterns). Lowercasing is Unicode-aware, so +// Cyrillic folds the same way Latin does. +func Tokens(utterance string) []string { + fields := strings.FieldsFunc(utterance, func(r rune) bool { + return r == ' ' || r == '\t' || r == '\n' || r == '\r' + }) + out := make([]string, 0, len(fields)) + for _, f := range fields { + t := strings.Trim(strings.ToLower(f), ".,!?;:()\"'«»…-–—") + if t == "" { + continue + } + out = append(out, t) + } + return out +} + +// Split partitions the utterance's tokens into the ones a claim explains and +// the rest, preserving order in both. A token is explained when it appears in +// one of the spans the claimant filled (a slot value, a matched substring). +// +// Duplicates are handled by membership and not by count: "напомни напомни +// позвонить" with span "напомни" explains both copies. The alternative is a +// multiset, and no claimant Maven has can say which copy it meant. +func Split(utterance string, spans ...string) (consumed, unexplained []string) { + explained := map[string]bool{} + for _, s := range spans { + for _, t := range Tokens(s) { + explained[t] = true + } + } + for _, t := range Tokens(utterance) { + if explained[t] { + consumed = append(consumed, t) + } else { + unexplained = append(unexplained, t) + } + } + return consumed, unexplained +} From 530c3ff395f9cdcf5fee8b117c3a9fecb185dac8 Mon Sep 17 00:00:00 2001 From: claude Date: Thu, 6 Aug 2026 00:54:43 +0400 Subject: [PATCH 4/5] claim tests: band order, coverage, the Rome case (V-565) TestCoverageBeatsBand is the V-558 failure as an ordering assertion. The weather claim explains the whole utterance and the pending reminder explains one token of it, so coverage settles it before the band is consulted. TestBandOrder asserts the order rather than trusting the iota, and pins BandUnknown at the bottom: a builder that forgot to set a band is a bug and must not outrank a measured claim. --- internal/claim/claim_test.go | 120 +++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 internal/claim/claim_test.go diff --git a/internal/claim/claim_test.go b/internal/claim/claim_test.go new file mode 100644 index 0000000..16c4310 --- /dev/null +++ b/internal/claim/claim_test.go @@ -0,0 +1,120 @@ +package claim + +import ( + "reflect" + "testing" +) + +func TestTokensStripsPunctuationAndCase(t *testing.T) { + got := Tokens("Какая сейчас погода в Риме?") + want := []string{"какая", "сейчас", "погода", "в", "риме"} + if !reflect.DeepEqual(got, want) { + t.Errorf("Tokens = %q, want %q", got, want) + } +} + +// The dash forms matter: an STT transcript routinely carries "а, да, прости - +// на 9", and a stray dash counted as a token would dilute every coverage score +// in the sentence. +func TestTokensDropsBareDashes(t *testing.T) { + got := Tokens("а, да, прости — на 9") + want := []string{"а", "да", "прости", "на", "9"} + if !reflect.DeepEqual(got, want) { + t.Errorf("Tokens = %q, want %q", got, want) + } +} + +func TestSplitPartitionsInOrder(t *testing.T) { + consumed, unexplained := Split("напомни позвонить маме", "позвонить маме") + if want := []string{"позвонить", "маме"}; !reflect.DeepEqual(consumed, want) { + t.Errorf("consumed = %q, want %q", consumed, want) + } + if want := []string{"напомни"}; !reflect.DeepEqual(unexplained, want) { + t.Errorf("unexplained = %q, want %q", unexplained, want) + } +} + +func TestCoverageIsZeroWithoutTokens(t *testing.T) { + if got := (Claim{}).Coverage(); got != 0 { + t.Errorf("Coverage of an empty claim = %v, want 0", got) + } +} + +func TestCoverageFraction(t *testing.T) { + c := Claim{Consumed: []string{"a", "b", "c"}, Unexplained: []string{"d"}} + if got := c.Coverage(); got != 0.75 { + t.Errorf("Coverage = %v, want 0.75", got) + } +} + +// The band order is load-bearing, so it is asserted rather than assumed from +// the iota. BandUnknown must sit at the bottom: a claim whose builder forgot to +// set a band is a bug and must not outrank a measured one. +func TestBandOrder(t *testing.T) { + ordered := []Band{BandUnknown, BandVetoed, BandNearest, BandStructural, BandAnchored} + for i := 1; i < len(ordered); i++ { + if !(ordered[i-1] < ordered[i]) { + t.Errorf("%v is not below %v", ordered[i-1], ordered[i]) + } + } + for _, b := range ordered { + if b.String() == "" { + t.Errorf("band %d has no name", b) + } + } + if BandUnknown.String() != "unknown" { + t.Errorf("BandUnknown.String() = %q", BandUnknown.String()) + } +} + +// The failure V-558 opened with, as an ordering test. A pending reminder eats +// "какая сейчас погода в Риме?" as a time answer and explains none of it; the +// weather source explains all of it. Coverage decides, and the band never gets +// consulted, which is the point: the pending claimant is structural and the +// weather claim is anchored, but even if the bands were equal the weather claim +// wins. +func TestCoverageBeatsBand(t *testing.T) { + utterance := "какая сейчас погода в Риме?" + pendingConsumed, pendingRest := Split(utterance, "сейчас") + pending := Claim{ + Claimant: "reminder-followup", Intent: "reminder", + Consumed: pendingConsumed, Unexplained: pendingRest, + Band: BandStructural, + } + weatherConsumed, weatherRest := Split(utterance, utterance) + weather := Claim{ + Claimant: "weather", Intent: "query", + Consumed: weatherConsumed, Unexplained: weatherRest, + Band: BandNearest, + } + if !weather.MoreSpecificThan(pending) { + t.Errorf("weather (%v cover) did not beat pending (%v cover)", + weather.Coverage(), pending.Coverage()) + } + if pending.MoreSpecificThan(weather) { + t.Error("pending beat weather, so the ordering is not asymmetric") + } +} + +// Where coverage ties, the band decides. Two grammars claiming the same +// utterance is the stage-0 contention case (ru-query-019 on the fixture), and +// today list order settles it with nothing recorded. +func TestBandBreaksACoverageTie(t *testing.T) { + anchored := Claim{Claimant: "calendar-query", Consumed: []string{"a"}, Band: BandAnchored} + nearest := Claim{Claimant: "classifier", Consumed: []string{"a"}, Band: BandNearest} + if !anchored.MoreSpecificThan(nearest) { + t.Error("anchored did not beat nearest on equal coverage") + } + if nearest.MoreSpecificThan(anchored) { + t.Error("nearest beat anchored on equal coverage") + } +} + +func TestVetoed(t *testing.T) { + if (Claim{}).Vetoed() { + t.Error("a claim with no veto reports itself vetoed") + } + if !(Claim{Veto: "fact with no key"}).Vetoed() { + t.Error("a claim with a veto reason does not report itself vetoed") + } +} From 4d942778366b572c6088975eb2c0ed4fbaa1cb18 Mon Sep 17 00:00:00 2001 From: claude Date: Thu, 6 Aug 2026 00:54:55 +0400 Subject: [PATCH 5/5] build a claim from a Decision, beside the existing path (V-565) router.ClaimOf maps a Decision onto the common unit. Stage 0 is anchored, the LLM path is structural, the classifier is nearest, and anything with a structural hole is vetoed whoever produced it. The veto recovers the reason gateLLMDecision throws away. Folding three named holes into llmThinConfidence leaves 0.3, which says something was wrong and never which thing, so the same conditions are read here as sentences a trace can print. Nothing in Route calls this. Decision.Confidence keeps its float and keeps working, because r.threshold and gateLLMDecision read it and the classifier is the failure floor. TestClaimOfLeavesTheDecisionAlone asserts that. TestONNXBaseline is unchanged at 64/91. --- internal/router/claim.go | 113 ++++++++++++++++++++++++ internal/router/claim_test.go | 160 ++++++++++++++++++++++++++++++++++ 2 files changed, 273 insertions(+) create mode 100644 internal/router/claim.go create mode 100644 internal/router/claim_test.go diff --git a/internal/router/claim.go b/internal/router/claim.go new file mode 100644 index 0000000..cbbb8ed --- /dev/null +++ b/internal/router/claim.go @@ -0,0 +1,113 @@ +package router + +import "github.com/kami/maven/internal/claim" + +// ClaimOf — build a claim.Claim from a Decision (V-565, design in +// docs/plans/19-dialogue-arbitration.md). +// +// Additive and beside the existing path. Decision.Confidence keeps its float +// and keeps working: r.threshold and gateLLMDecision read it, and the +// classifier cascade is the failure floor. Nothing in Route calls this yet. +// The arbiter that reads claims is V-560. +// +// claimant names who produced the decision. The cascade does not record which +// stage-0 grammar matched, so the caller passes what it knows and the builder +// does not guess. +func ClaimOf(claimant string, d Decision) claim.Claim { + consumed, unexplained := claim.Split(d.Utterance, claimSpans(d)...) + return claim.Claim{ + Claimant: claimant, + Intent: string(d.Intent), + Filled: filledSlots(d.Slots), + Consumed: consumed, + Unexplained: unexplained, + Band: bandOf(d), + Veto: vetoOf(d), + } +} + +// claimSpans — the parts of the utterance the decision says it read. Slot +// values, not the utterance, because coverage is the question of how much of +// the sentence the claim actually explains. +// +// A stage-0 grammar reports whatever its Build put in the slots, which for the +// reminder rule is the text after "напомни" and not the verb itself. That +// under-reports coverage rather than over-reporting it, which is the safe +// direction: a claim that overstates what it explains wins arbitrations it +// should lose. +func claimSpans(d Decision) []string { + spans := []string{d.Slots.Text, d.Slots.Key, d.Slots.Value, d.Slots.Fn} + return append(spans, d.Slots.Args...) +} + +// filledSlots — the slot names this decision would fill. Text counts only when +// it differs from the whole utterance: fillSlots backfills the raw utterance +// into Text for a note, a query and a chat turn, so a set Text is not by itself +// evidence that anything was extracted. +func filledSlots(s Slots) []string { + var out []string + if s.HasTime { + out = append(out, "time") + } + if s.HasFn { + out = append(out, "fn") + } + if s.HasKey { + out = append(out, "key") + } + if s.Text != "" { + out = append(out, "text") + } + return out +} + +// bandOf — which kind of evidence this decision rests on. +// +// Stage 0 is anchored: a literal pattern matched and its span decided the +// intent. The LLM path (stage 1) is structural: the model read the whole +// sentence, and gateLLMDecision already checked the route for structural +// holes. The classifier (stages 2 and 3) is nearest, and the measurement is why +// it is one band rather than a scale — on the 91-case RU fixture its cosine +// spans 0.859 to 0.942 and scores 62% at both ends. +// +// A decision carrying a veto lands in BandVetoed regardless of who produced it. +// That is the point of the band: a self-vetoed claim should lose to any claim +// that is not, whatever machinery built it. +func bandOf(d Decision) claim.Band { + if vetoOf(d) != "" { + return claim.BandVetoed + } + switch d.Stage { + case 0: + return claim.BandAnchored + case 1: + return claim.BandStructural + default: + return claim.BandNearest + } +} + +// vetoOf — why this decision should not win, recovered as a reason rather than +// a number. +// +// gateLLMDecision flattens three named structural holes into +// llmThinConfidence, and the reason is lost at that point: 0.3 tells a reader +// that something was wrong and never which thing. The same three conditions are +// checked here so the claim carries the sentence a trace can print and the +// owner can be told. +// +// Clarify is checked last and is the general case. A decision below threshold +// has already asked to be doubted, whichever path set it. +func vetoOf(d Decision) string { + switch { + case d.Intent == IntentFact && !d.Slots.HasKey: + return "fact with no key: nothing to write, or a confident write under the wrong key" + case d.Intent == IntentAct && !d.Slots.HasFn: + return "act with no allowlisted fn: running an unlisted command or silently doing nothing" + case d.Intent == IntentReminder && !reminderHasSubject(d.Slots.Text): + return "reminder with no subject: it would fire empty at the hour" + case d.Clarify: + return "below the confidence gate" + } + return "" +} diff --git a/internal/router/claim_test.go b/internal/router/claim_test.go new file mode 100644 index 0000000..e9dd784 --- /dev/null +++ b/internal/router/claim_test.go @@ -0,0 +1,160 @@ +package router + +import ( + "context" + "reflect" + "testing" + "time" + + "github.com/kami/maven/internal/claim" +) + +func TestClaimOfBands(t *testing.T) { + cases := []struct { + name string + dec Decision + want claim.Band + }{ + { + name: "stage 0 is anchored", + dec: Decision{ + Utterance: "сколько времени", Stage: 0, Intent: IntentSystem, + Confidence: 1.0, Slots: Slots{Text: "сколько времени"}, + }, + want: claim.BandAnchored, + }, + { + name: "the llm path is structural", + dec: Decision{ + Utterance: "выпил воды", Stage: 1, Intent: IntentFact, + Confidence: llmFullConfidence, + Slots: Slots{Key: "water", Value: "1", HasKey: true, Text: "выпил воды"}, + }, + want: claim.BandStructural, + }, + { + name: "the classifier is nearest", + dec: Decision{ + Utterance: "что нового", Stage: 2, Intent: IntentQuery, + Confidence: 0.91, Slots: Slots{Text: "что нового"}, + }, + want: claim.BandNearest, + }, + { + name: "a structural hole vetoes whoever found it", + dec: Decision{ + Utterance: "запиши", Stage: 1, Intent: IntentFact, + Confidence: llmThinConfidence, Slots: Slots{Text: "запиши"}, + }, + want: claim.BandVetoed, + }, + { + name: "clarify vetoes a classifier decision", + dec: Decision{ + Utterance: "сделай это", Stage: 3, Intent: IntentNote, + Confidence: 0.2, Clarify: true, Slots: Slots{Text: "сделай это"}, + }, + want: claim.BandVetoed, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := ClaimOf("test", tc.dec) + if got.Band != tc.want { + t.Errorf("band = %v, want %v (veto %q)", got.Band, tc.want, got.Veto) + } + }) + } +} + +// The veto has to name the hole. Folding all three arms into llmThinConfidence +// is what lost the reason, and 0.3 tells a reader that something was wrong but +// never which thing. +func TestClaimOfVetoNamesTheHole(t *testing.T) { + cases := []struct { + name string + dec Decision + want string + }{ + {"keyless fact", Decision{Intent: IntentFact}, "fact with no key"}, + {"act with no fn", Decision{Intent: IntentAct}, "act with no allowlisted fn"}, + {"subjectless reminder", Decision{Intent: IntentReminder, Slots: Slots{Text: "напомни"}}, "reminder with no subject"}, + {"below the gate", Decision{Intent: IntentQuery, Clarify: true}, "below the confidence gate"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := ClaimOf("test", tc.dec) + if !got.Vetoed() { + t.Fatalf("no veto, want one about %q", tc.want) + } + if len(got.Veto) < len(tc.want) || got.Veto[:len(tc.want)] != tc.want { + t.Errorf("veto = %q, want it to start with %q", got.Veto, tc.want) + } + }) + } +} + +// A route with every slot filled must NOT be vetoed. The three arms are +// structural holes, not a tax on every decision. +func TestClaimOfCompleteRouteIsNotVetoed(t *testing.T) { + d := Decision{ + Utterance: "напомни позвонить маме в семь", Stage: 1, Intent: IntentReminder, + Confidence: llmFullConfidence, + Slots: Slots{Text: "позвонить маме", Time: time.Now(), HasTime: true}, + } + c := ClaimOf("llm", d) + if c.Vetoed() { + t.Errorf("complete reminder vetoed: %q", c.Veto) + } + if c.Band != claim.BandStructural { + t.Errorf("band = %v, want structural", c.Band) + } +} + +func TestClaimOfCoverageAndFilledSlots(t *testing.T) { + d := Decision{ + Utterance: "напомни позвонить маме", Stage: 0, Intent: IntentReminder, + Confidence: 1.0, Slots: Slots{Text: "позвонить маме"}, + } + c := ClaimOf("reminder-wakeword", d) + // The grammar captures what follows the verb, so "напомни" itself is + // unexplained. Under-reporting is the safe direction. + if len(c.Consumed) != 2 || len(c.Unexplained) != 1 { + t.Errorf("consumed %q / unexplained %q, want 2 and 1", c.Consumed, c.Unexplained) + } + if got := c.Coverage(); got < 0.66 || got > 0.67 { + t.Errorf("coverage = %v, want about 2/3", got) + } + if c.Intent != string(IntentReminder) { + t.Errorf("intent = %q", c.Intent) + } + if len(c.Filled) != 1 || c.Filled[0] != "text" { + t.Errorf("filled = %q, want [text]", c.Filled) + } + if c.Claimant != "reminder-wakeword" { + t.Errorf("claimant = %q", c.Claimant) + } +} + +// The point of V-565's "additive" constraint, asserted rather than trusted: +// building a claim reads a Decision and changes nothing about it, so the +// classifier floor and the two consumers of Confidence are untouched. +func TestClaimOfLeavesTheDecisionAlone(t *testing.T) { + r := New(Config{ + Grammars: []Grammar{ReminderGrammar()}, + Extractor: Extractor{}, + Threshold: 0.55, + }) + before, err := r.Route(context.Background(), "напомни полить цветы", time.Now()) + if err != nil { + t.Fatalf("Route: %v", err) + } + after := before + _ = ClaimOf("reminder-wakeword", after) + if !reflect.DeepEqual(after, before) { + t.Errorf("ClaimOf mutated the decision: %+v vs %+v", after, before) + } + if before.Confidence != 1.0 { + t.Errorf("stage 0 confidence = %v, want 1.0 — the float still has to work", before.Confidence) + } +}