measure what each claimant on an utterance reports (V-565)

Two reporting tests over the 91-case RU fixture, no ratchet: a ratchet here
would freeze a number nobody has decided to hold.

TestStage0Contention runs the 21 grammars one at a time instead of stopping
at the first match. One case of 91 draws two, ru-query-019, where
calendar-query beats agenda-query by list position alone.

TestONNXClaimConfidenceDistribution buckets the reported confidence by the
layer that produced it. Stage 0 is 20/20 at a hardcoded 1.0. The classifier
scores 62% below its median and 62% above, across a cosine range of 0.859
to 0.942, with a top-two margin of p50 0.009. The float is not a confidence.

newBaselineClassifier and baselineGrammars split out of newBaselineRouter so
the measurement runs the same rules the daemon runs. TestONNXBaseline is
unchanged at 64/91.
This commit is contained in:
2026-08-06 00:54:13 +04:00
parent da2d11dab6
commit 934a28d67c
2 changed files with 261 additions and 13 deletions
+31 -13
View File
@@ -216,6 +216,27 @@ func TestONNXBaseline(t *testing.T) {
func newBaselineRouter(t *testing.T, emb router.Embedder, llmR *router.LLMRouter) *router.Router {
t.Helper()
acts := router.DefaultActMatcher{Fns: actFns}
cls := newBaselineClassifier(t, emb)
return router.New(router.Config{
Grammars: baselineGrammars(acts),
Classifier: cls,
Extractor: router.Extractor{
Time: router.StubDateTimeParser{},
Acts: acts,
Facts: router.DefaultFactParser{},
},
// The deployed gate, not a test-local one: a fixture scored at a looser
// threshold reports an accuracy no real turn would see.
Threshold: config.DefaultRouterThreshold,
LLM: llmR,
})
}
// newBaselineClassifier — the seeded nearest-centroid classifier the cascade
// runs. Split out of newBaselineRouter so the claim measurement can ask it for
// its full ranking, not just the winner the Decision carries.
func newBaselineClassifier(t *testing.T, emb router.Embedder) *router.Classifier {
t.Helper()
cls := router.NewClassifier(emb)
ctx := context.Background()
seeds := seedsWithIntent(t)
@@ -231,6 +252,15 @@ func newBaselineRouter(t *testing.T, emb router.Embedder, llmR *router.LLMRouter
t.Fatalf("seed %q: %v", text, err)
}
}
return cls
}
// baselineGrammars — the stage-0 rule set in the daemon's order (buildRouter in
// cmd/mavend/voicewire.go). Split out of newBaselineRouter so the claim
// measurement can run the same rules one at a time and see which of them
// contend for the same utterance, which the cascade hides by stopping at the
// first match.
func baselineGrammars(acts router.ActMatcher) []router.Grammar {
grammars := router.DefaultGrammars(acts)
grammars = append(grammars, router.SystemTimeDateGrammars()...)
// Same order as buildRouter (voicewire.go). The fixture is only worth
@@ -248,19 +278,7 @@ func newBaselineRouter(t *testing.T, emb router.Embedder, llmR *router.LLMRouter
// "расскажи про X" is a world question the model called a fact, and the
// rule goes last because it matches on the first word alone (Vikunja #498).
grammars = append(grammars, router.NarrativeQueryGrammars()...)
return router.New(router.Config{
Grammars: grammars,
Classifier: cls,
Extractor: router.Extractor{
Time: router.StubDateTimeParser{},
Acts: acts,
Facts: router.DefaultFactParser{},
},
// The deployed gate, not a test-local one: a fixture scored at a looser
// threshold reports an accuracy no real turn would see.
Threshold: config.DefaultRouterThreshold,
LLM: llmR,
})
return grammars
}
// seedOrder — fixed iteration order over the corpus. Not cosmetic: a few