measure what each claimant on an utterance reports (V-565)
Two reporting tests over the 91-case RU fixture, no ratchet: a ratchet here would freeze a number nobody has decided to hold. TestStage0Contention runs the 21 grammars one at a time instead of stopping at the first match. One case of 91 draws two, ru-query-019, where calendar-query beats agenda-query by list position alone. TestONNXClaimConfidenceDistribution buckets the reported confidence by the layer that produced it. Stage 0 is 20/20 at a hardcoded 1.0. The classifier scores 62% below its median and 62% above, across a cosine range of 0.859 to 0.942, with a top-two margin of p50 0.009. The float is not a confidence. newBaselineClassifier and baselineGrammars split out of newBaselineRouter so the measurement runs the same rules the daemon runs. TestONNXBaseline is unchanged at 64/91.
This commit is contained in:
@@ -216,6 +216,27 @@ func TestONNXBaseline(t *testing.T) {
|
||||
func newBaselineRouter(t *testing.T, emb router.Embedder, llmR *router.LLMRouter) *router.Router {
|
||||
t.Helper()
|
||||
acts := router.DefaultActMatcher{Fns: actFns}
|
||||
cls := newBaselineClassifier(t, emb)
|
||||
return router.New(router.Config{
|
||||
Grammars: baselineGrammars(acts),
|
||||
Classifier: cls,
|
||||
Extractor: router.Extractor{
|
||||
Time: router.StubDateTimeParser{},
|
||||
Acts: acts,
|
||||
Facts: router.DefaultFactParser{},
|
||||
},
|
||||
// The deployed gate, not a test-local one: a fixture scored at a looser
|
||||
// threshold reports an accuracy no real turn would see.
|
||||
Threshold: config.DefaultRouterThreshold,
|
||||
LLM: llmR,
|
||||
})
|
||||
}
|
||||
|
||||
// newBaselineClassifier — the seeded nearest-centroid classifier the cascade
|
||||
// runs. Split out of newBaselineRouter so the claim measurement can ask it for
|
||||
// its full ranking, not just the winner the Decision carries.
|
||||
func newBaselineClassifier(t *testing.T, emb router.Embedder) *router.Classifier {
|
||||
t.Helper()
|
||||
cls := router.NewClassifier(emb)
|
||||
ctx := context.Background()
|
||||
seeds := seedsWithIntent(t)
|
||||
@@ -231,6 +252,15 @@ func newBaselineRouter(t *testing.T, emb router.Embedder, llmR *router.LLMRouter
|
||||
t.Fatalf("seed %q: %v", text, err)
|
||||
}
|
||||
}
|
||||
return cls
|
||||
}
|
||||
|
||||
// baselineGrammars — the stage-0 rule set in the daemon's order (buildRouter in
|
||||
// cmd/mavend/voicewire.go). Split out of newBaselineRouter so the claim
|
||||
// measurement can run the same rules one at a time and see which of them
|
||||
// contend for the same utterance, which the cascade hides by stopping at the
|
||||
// first match.
|
||||
func baselineGrammars(acts router.ActMatcher) []router.Grammar {
|
||||
grammars := router.DefaultGrammars(acts)
|
||||
grammars = append(grammars, router.SystemTimeDateGrammars()...)
|
||||
// Same order as buildRouter (voicewire.go). The fixture is only worth
|
||||
@@ -248,19 +278,7 @@ func newBaselineRouter(t *testing.T, emb router.Embedder, llmR *router.LLMRouter
|
||||
// "расскажи про X" is a world question the model called a fact, and the
|
||||
// rule goes last because it matches on the first word alone (Vikunja #498).
|
||||
grammars = append(grammars, router.NarrativeQueryGrammars()...)
|
||||
return router.New(router.Config{
|
||||
Grammars: grammars,
|
||||
Classifier: cls,
|
||||
Extractor: router.Extractor{
|
||||
Time: router.StubDateTimeParser{},
|
||||
Acts: acts,
|
||||
Facts: router.DefaultFactParser{},
|
||||
},
|
||||
// The deployed gate, not a test-local one: a fixture scored at a looser
|
||||
// threshold reports an accuracy no real turn would see.
|
||||
Threshold: config.DefaultRouterThreshold,
|
||||
LLM: llmR,
|
||||
})
|
||||
return grammars
|
||||
}
|
||||
|
||||
// seedOrder — fixed iteration order over the corpus. Not cosmetic: a few
|
||||
|
||||
Reference in New Issue
Block a user