Maven/internal/router/semantic/eval_test.go

package semantic

import (
	"context"
	"testing"
	"time"

	"github.com/kami/maven/internal/router"
)

// SemanticRouterFunc adapts a bare function to SemanticRouter.
type SemanticRouterFunc func(ctx context.Context, text string) (SemanticRouteDecision, error)

func (f SemanticRouterFunc) Route(ctx context.Context, text string) (SemanticRouteDecision, error) {
	return f(ctx, text)
}

func TestEvalScoring(t *testing.T) {
	model := SemanticRouterFunc(func(ctx context.Context, text string) (SemanticRouteDecision, error) {
		return SemanticRouteDecision{Route: RouteKnowledge, Confidence: 0.9}, nil
	})

	evalSet := []EvalCase{
		{ID: "a1", Text: "привет", ExpectedRoute: RouteConversation},
		{ID: "a2", Text: "сколько воды", ExpectedRoute: RouteKnowledge},
		{ID: "a3", Text: "выключи свет", ExpectedRoute: RouteAction},
		{ID: "a4", Text: "запиши заметку", ExpectedRoute: RouteMemoryWrite},
	}

	rep := ScoreEval(model, evalSet)
	if rep.Total != 4 {
		t.Errorf("Total = %d, want 4", rep.Total)
	}
	if rep.Passed != 1 {
		t.Errorf("Passed = %d, want 1 (only knowledge)", rep.Passed)
	}
	if rep.FalseAction != 0 {
		t.Errorf("FalseAction = %d, want 0", rep.FalseAction)
	}
	km := rep.ByRoute[RouteKnowledge]
	if km.Precision != 0.25 {
		t.Errorf("knowledge precision = %.3f, want 0.250", km.Precision)
	}
	if km.Recall != 1.0 {
		t.Errorf("knowledge recall = %.3f, want 1.000", km.Recall)
	}
	t.Logf("eval report:\n%s", rep.String())
}

func TestShadowHarness(t *testing.T) {
	model := SemanticRouterFunc(func(ctx context.Context, text string) (SemanticRouteDecision, error) {
		if text == "выключи свет" {
			return SemanticRouteDecision{Route: RouteAction, Confidence: 0.9}, nil
		}
		return SemanticRouteDecision{Route: RouteConversation, Confidence: 0.7}, nil
	})

	h := NewShadowHarness(model)

	h.Observe(context.Background(), "выключи свет",
		router.Decision{Intent: router.IntentAct}, false)
	h.Observe(context.Background(), "привет",
		router.Decision{Intent: router.IntentChat}, false)
	h.Observe(context.Background(), "перезапусти докер",
		router.Decision{Intent: router.IntentAct}, true)

	rep := h.Summarize()
	if rep.Total != 3 {
		t.Errorf("Total = %d, want 3", rep.Total)
	}
	if rep.Agree != 2 {
		t.Errorf("Agree = %d, want 2", rep.Agree)
	}
	if rep.Disagree != 1 {
		t.Errorf("Disagree = %d, want 1", rep.Disagree)
	}
	if rep.FastPathTotal != 1 {
		t.Errorf("FastPathTotal = %d, want 1", rep.FastPathTotal)
	}
	if rep.ResidualTotal != 2 {
		t.Errorf("ResidualTotal = %d, want 2", rep.ResidualTotal)
	}
	t.Logf("shadow report:\n%s", rep.String())
}

func TestShadowHarnessNilModel(t *testing.T) {
	h := NewShadowHarness(nil)
	h.Observe(context.Background(), "привет",
		router.Decision{Intent: router.IntentChat}, false)
	rep := h.Summarize()
	if rep.Total != 1 {
		t.Errorf("Total = %d, want 1", rep.Total)
	}
	if rep.Agree != 0 {
		t.Errorf("Agree = %d, want 0 (nil model → uncertain)", rep.Agree)
	}
}

// legacyRouterAdapter wraps a *router.Router to satisfy LegacyRouter.
type legacyRouterAdapter struct {
	router *router.Router
}

func (a *legacyRouterAdapter) Route(ctx context.Context, input router.NormalizedInput, now time.Time) (router.Decision, error) {
	return a.router.Route(ctx, input, now)
}

// TestLegacyBaseline runs the actual router cascade against the corpus and
// reports the real baseline. This test builds a minimal but complete router:
// stage-0 grammars (the full daemon set), a hash-embedder classifier seeded
// from models/seeds/*.txt, and the deployed confidence threshold.
//
// The hash embedder is deterministic, so this baseline is reproducible. The
// ONNX embedder would score higher; measure both before drawing conclusions.
func TestLegacyBaseline(t *testing.T) {
	exs, err := LoadCorpus()
	if err != nil {
		t.Fatal(err)
	}

	r := buildMinimalRouter(t)
	now := time.Now()

	rep := ScoreLegacy(context.Background(), r, exs, now)
	t.Log("\n" + rep.String())

	// Print contrast family breakdown.
	families := ContrastFamilies(rep)
	if len(families) > 0 {
		t.Logf("contrast families:\n%s", ContrastFamilyReportString(families))
	}

	// Log residual-only metrics.
	t.Logf("residual: %d/%d (%.1f%%)",
		rep.ResidualPassed, rep.ResidualTotal,
		100*float64(rep.ResidualPassed)/maxf(float64(rep.ResidualTotal), 1))
}

// buildMinimalRouter creates a router with the daemon's grammar set, a
// hash-embedder classifier seeded from models/seeds, and the deployed
// threshold. This is the minimal real router that can score the corpus.
func buildMinimalRouter(t *testing.T) LegacyRouter {
	t.Helper()
	// Use the same act matcher and seed loading as the eval package.
	acts := router.DefaultActMatcher{Fns: actVerbList()}
	cls := buildSeededClassifier(t, router.NewHashEmbedder(1024))
	r := router.New(router.Config{
		Grammars:   router.StageZeroGrammars(acts),
		Classifier: cls,
		Extractor: router.Extractor{
			Time:  router.StubDateTimeParser{},
			Acts:  acts,
			Facts: router.DefaultFactParser{},
		},
		Threshold: 0.55,
	})
	return &legacyRouterAdapter{router: r}
}

func maxf(a, b float64) float64 {
	if a > b {
		return a
	}
	return b
}