package semantic import ( "context" "testing" "time" "github.com/kami/maven/internal/router" ) // SemanticRouterFunc adapts a bare function to SemanticRouter. type SemanticRouterFunc func(ctx context.Context, text string) (SemanticRouteDecision, error) func (f SemanticRouterFunc) Route(ctx context.Context, text string) (SemanticRouteDecision, error) { return f(ctx, text) } func TestEvalScoring(t *testing.T) { model := SemanticRouterFunc(func(ctx context.Context, text string) (SemanticRouteDecision, error) { return SemanticRouteDecision{Route: RouteKnowledge, Confidence: 0.9}, nil }) evalSet := []EvalCase{ {ID: "a1", Text: "привет", ExpectedRoute: RouteConversation}, {ID: "a2", Text: "сколько воды", ExpectedRoute: RouteKnowledge}, {ID: "a3", Text: "выключи свет", ExpectedRoute: RouteAction}, {ID: "a4", Text: "запиши заметку", ExpectedRoute: RouteMemoryWrite}, } rep := ScoreEval(model, evalSet) if rep.Total != 4 { t.Errorf("Total = %d, want 4", rep.Total) } if rep.Passed != 1 { t.Errorf("Passed = %d, want 1 (only knowledge)", rep.Passed) } if rep.FalseAction != 0 { t.Errorf("FalseAction = %d, want 0", rep.FalseAction) } km := rep.ByRoute[RouteKnowledge] if km.Precision != 0.25 { t.Errorf("knowledge precision = %.3f, want 0.250", km.Precision) } if km.Recall != 1.0 { t.Errorf("knowledge recall = %.3f, want 1.000", km.Recall) } t.Logf("eval report:\n%s", rep.String()) } func TestShadowHarness(t *testing.T) { model := SemanticRouterFunc(func(ctx context.Context, text string) (SemanticRouteDecision, error) { if text == "выключи свет" { return SemanticRouteDecision{Route: RouteAction, Confidence: 0.9}, nil } return SemanticRouteDecision{Route: RouteConversation, Confidence: 0.7}, nil }) h := NewShadowHarness(model) h.Observe(context.Background(), "выключи свет", router.Decision{Intent: router.IntentAct}, false) h.Observe(context.Background(), "привет", router.Decision{Intent: router.IntentChat}, false) h.Observe(context.Background(), "перезапусти докер", router.Decision{Intent: router.IntentAct}, true) rep := h.Summarize() if rep.Total != 3 { t.Errorf("Total = %d, want 3", rep.Total) } if rep.Agree != 2 { t.Errorf("Agree = %d, want 2", rep.Agree) } if rep.Disagree != 1 { t.Errorf("Disagree = %d, want 1", rep.Disagree) } if rep.FastPathTotal != 1 { t.Errorf("FastPathTotal = %d, want 1", rep.FastPathTotal) } if rep.ResidualTotal != 2 { t.Errorf("ResidualTotal = %d, want 2", rep.ResidualTotal) } t.Logf("shadow report:\n%s", rep.String()) } func TestShadowHarnessNilModel(t *testing.T) { h := NewShadowHarness(nil) h.Observe(context.Background(), "привет", router.Decision{Intent: router.IntentChat}, false) rep := h.Summarize() if rep.Total != 1 { t.Errorf("Total = %d, want 1", rep.Total) } if rep.Agree != 0 { t.Errorf("Agree = %d, want 0 (nil model → uncertain)", rep.Agree) } } // legacyRouterAdapter wraps a *router.Router to satisfy LegacyRouter. type legacyRouterAdapter struct { router *router.Router } func (a *legacyRouterAdapter) Route(ctx context.Context, input router.NormalizedInput, now time.Time) (router.Decision, error) { return a.router.Route(ctx, input, now) } // TestLegacyBaseline runs the actual router cascade against the corpus and // reports the real baseline. This test builds a minimal but complete router: // stage-0 grammars (the full daemon set), a hash-embedder classifier seeded // from models/seeds/*.txt, and the deployed confidence threshold. // // The hash embedder is deterministic, so this baseline is reproducible. The // ONNX embedder would score higher; measure both before drawing conclusions. func TestLegacyBaseline(t *testing.T) { exs, err := LoadCorpus() if err != nil { t.Fatal(err) } r := buildMinimalRouter(t) now := time.Now() rep := ScoreLegacy(context.Background(), r, exs, now) t.Log("\n" + rep.String()) // Print contrast family breakdown. families := ContrastFamilies(rep) if len(families) > 0 { t.Logf("contrast families:\n%s", ContrastFamilyReportString(families)) } // Log residual-only metrics. t.Logf("residual: %d/%d (%.1f%%)", rep.ResidualPassed, rep.ResidualTotal, 100*float64(rep.ResidualPassed)/maxf(float64(rep.ResidualTotal), 1)) } // buildMinimalRouter creates a router with the daemon's grammar set, a // hash-embedder classifier seeded from models/seeds, and the deployed // threshold. This is the minimal real router that can score the corpus. func buildMinimalRouter(t *testing.T) LegacyRouter { t.Helper() // Use the same act matcher and seed loading as the eval package. acts := router.DefaultActMatcher{Fns: actVerbList()} cls := buildSeededClassifier(t, router.NewHashEmbedder(1024)) r := router.New(router.Config{ Grammars: router.StageZeroGrammars(acts), Classifier: cls, Extractor: router.Extractor{ Time: router.StubDateTimeParser{}, Acts: acts, Facts: router.DefaultFactParser{}, }, Threshold: 0.55, }) return &legacyRouterAdapter{router: r} } func maxf(a, b float64) float64 { if a > b { return a } return b }