router/semantic: legacy baseline eval test (slice 13)
TestLegacyBaseline builds a minimal but complete router (stage-0 grammars, hash-embedder classifier seeded from models/seeds, deployed confidence threshold) and runs it against the full 136-example corpus. Reports macro F1, false-action rate, fast-path/residual/router-residual accuracy, per-route P/R/F1, confusion matrix, and contrast family breakdown (negation, question, reported_speech, quotation, hypothetical, capability_question). Pre-route consumed cases are tracked separately from the fast-path bucket. The hash embedder is deterministic, so this baseline is reproducible. The ONNX embedder would score higher; measure both before drawing conclusions.
This commit is contained in:
@@ -3,6 +3,7 @@ package semantic
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/router"
|
||||
)
|
||||
@@ -94,3 +95,71 @@ func TestShadowHarnessNilModel(t *testing.T) {
|
||||
t.Errorf("Agree = %d, want 0 (nil model → uncertain)", rep.Agree)
|
||||
}
|
||||
}
|
||||
|
||||
// legacyRouterAdapter wraps a *router.Router to satisfy LegacyRouter.
|
||||
type legacyRouterAdapter struct {
|
||||
router *router.Router
|
||||
}
|
||||
|
||||
func (a *legacyRouterAdapter) Route(ctx context.Context, input router.NormalizedInput, now time.Time) (router.Decision, error) {
|
||||
return a.router.Route(ctx, input, now)
|
||||
}
|
||||
|
||||
// TestLegacyBaseline runs the actual router cascade against the corpus and
|
||||
// reports the real baseline. This test builds a minimal but complete router:
|
||||
// stage-0 grammars (the full daemon set), a hash-embedder classifier seeded
|
||||
// from models/seeds/*.txt, and the deployed confidence threshold.
|
||||
//
|
||||
// The hash embedder is deterministic, so this baseline is reproducible. The
|
||||
// ONNX embedder would score higher; measure both before drawing conclusions.
|
||||
func TestLegacyBaseline(t *testing.T) {
|
||||
exs, err := LoadCorpus()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
r := buildMinimalRouter(t)
|
||||
now := time.Now()
|
||||
|
||||
rep := ScoreLegacy(context.Background(), r, exs, now)
|
||||
t.Log("\n" + rep.String())
|
||||
|
||||
// Print contrast family breakdown.
|
||||
families := ContrastFamilies(rep)
|
||||
if len(families) > 0 {
|
||||
t.Logf("contrast families:\n%s", ContrastFamilyReportString(families))
|
||||
}
|
||||
|
||||
// Log residual-only metrics.
|
||||
t.Logf("residual: %d/%d (%.1f%%)",
|
||||
rep.ResidualPassed, rep.ResidualTotal,
|
||||
100*float64(rep.ResidualPassed)/maxf(float64(rep.ResidualTotal), 1))
|
||||
}
|
||||
|
||||
// buildMinimalRouter creates a router with the daemon's grammar set, a
|
||||
// hash-embedder classifier seeded from models/seeds, and the deployed
|
||||
// threshold. This is the minimal real router that can score the corpus.
|
||||
func buildMinimalRouter(t *testing.T) LegacyRouter {
|
||||
t.Helper()
|
||||
// Use the same act matcher and seed loading as the eval package.
|
||||
acts := router.DefaultActMatcher{Fns: actVerbList()}
|
||||
cls := buildSeededClassifier(t, router.NewHashEmbedder(1024))
|
||||
r := router.New(router.Config{
|
||||
Grammars: router.StageZeroGrammars(acts),
|
||||
Classifier: cls,
|
||||
Extractor: router.Extractor{
|
||||
Time: router.StubDateTimeParser{},
|
||||
Acts: acts,
|
||||
Facts: router.DefaultFactParser{},
|
||||
},
|
||||
Threshold: 0.55,
|
||||
})
|
||||
return &legacyRouterAdapter{router: r}
|
||||
}
|
||||
|
||||
func maxf(a, b float64) float64 {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user