router/semantic: legacy baseline eval test (slice 13)

TestLegacyBaseline builds a minimal but complete router (stage-0
grammars, hash-embedder classifier seeded from models/seeds, deployed
confidence threshold) and runs it against the full 136-example corpus.

Reports macro F1, false-action rate, fast-path/residual/router-residual
accuracy, per-route P/R/F1, confusion matrix, and contrast family
breakdown (negation, question, reported_speech, quotation, hypothetical,
capability_question). Pre-route consumed cases are tracked separately
from the fast-path bucket.

The hash embedder is deterministic, so this baseline is reproducible.
The ONNX embedder would score higher; measure both before drawing
conclusions.
This commit is contained in:
2026-09-07 02:09:02 +04:00
parent 1b8ae3c3e5
commit 59a0a08d32
+69
View File
@@ -3,6 +3,7 @@ package semantic
import (
"context"
"testing"
"time"
"github.com/kami/maven/internal/router"
)
@@ -94,3 +95,71 @@ func TestShadowHarnessNilModel(t *testing.T) {
t.Errorf("Agree = %d, want 0 (nil model → uncertain)", rep.Agree)
}
}
// legacyRouterAdapter wraps a *router.Router to satisfy LegacyRouter.
type legacyRouterAdapter struct {
router *router.Router
}
func (a *legacyRouterAdapter) Route(ctx context.Context, input router.NormalizedInput, now time.Time) (router.Decision, error) {
return a.router.Route(ctx, input, now)
}
// TestLegacyBaseline runs the actual router cascade against the corpus and
// reports the real baseline. This test builds a minimal but complete router:
// stage-0 grammars (the full daemon set), a hash-embedder classifier seeded
// from models/seeds/*.txt, and the deployed confidence threshold.
//
// The hash embedder is deterministic, so this baseline is reproducible. The
// ONNX embedder would score higher; measure both before drawing conclusions.
func TestLegacyBaseline(t *testing.T) {
exs, err := LoadCorpus()
if err != nil {
t.Fatal(err)
}
r := buildMinimalRouter(t)
now := time.Now()
rep := ScoreLegacy(context.Background(), r, exs, now)
t.Log("\n" + rep.String())
// Print contrast family breakdown.
families := ContrastFamilies(rep)
if len(families) > 0 {
t.Logf("contrast families:\n%s", ContrastFamilyReportString(families))
}
// Log residual-only metrics.
t.Logf("residual: %d/%d (%.1f%%)",
rep.ResidualPassed, rep.ResidualTotal,
100*float64(rep.ResidualPassed)/maxf(float64(rep.ResidualTotal), 1))
}
// buildMinimalRouter creates a router with the daemon's grammar set, a
// hash-embedder classifier seeded from models/seeds, and the deployed
// threshold. This is the minimal real router that can score the corpus.
func buildMinimalRouter(t *testing.T) LegacyRouter {
t.Helper()
// Use the same act matcher and seed loading as the eval package.
acts := router.DefaultActMatcher{Fns: actVerbList()}
cls := buildSeededClassifier(t, router.NewHashEmbedder(1024))
r := router.New(router.Config{
Grammars: router.StageZeroGrammars(acts),
Classifier: cls,
Extractor: router.Extractor{
Time: router.StubDateTimeParser{},
Acts: acts,
Facts: router.DefaultFactParser{},
},
Threshold: 0.55,
})
return &legacyRouterAdapter{router: r}
}
func maxf(a, b float64) float64 {
if a > b {
return a
}
return b
}