97 lines
2.9 KiB
Go
97 lines
2.9 KiB
Go
package semantic
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
|
|
"github.com/kami/maven/internal/router"
|
|
)
|
|
|
|
// SemanticRouterFunc adapts a bare function to SemanticRouter.
|
|
type SemanticRouterFunc func(ctx context.Context, text string) (SemanticRouteDecision, error)
|
|
|
|
func (f SemanticRouterFunc) Route(ctx context.Context, text string) (SemanticRouteDecision, error) {
|
|
return f(ctx, text)
|
|
}
|
|
|
|
func TestEvalScoring(t *testing.T) {
|
|
model := SemanticRouterFunc(func(ctx context.Context, text string) (SemanticRouteDecision, error) {
|
|
return SemanticRouteDecision{Route: RouteKnowledge, Confidence: 0.9}, nil
|
|
})
|
|
|
|
evalSet := []EvalCase{
|
|
{ID: "a1", Text: "привет", ExpectedRoute: RouteConversation},
|
|
{ID: "a2", Text: "сколько воды", ExpectedRoute: RouteKnowledge},
|
|
{ID: "a3", Text: "выключи свет", ExpectedRoute: RouteAction},
|
|
{ID: "a4", Text: "запиши заметку", ExpectedRoute: RouteMemoryWrite},
|
|
}
|
|
|
|
rep := ScoreEval(model, evalSet)
|
|
if rep.Total != 4 {
|
|
t.Errorf("Total = %d, want 4", rep.Total)
|
|
}
|
|
if rep.Passed != 1 {
|
|
t.Errorf("Passed = %d, want 1 (only knowledge)", rep.Passed)
|
|
}
|
|
if rep.FalseAction != 0 {
|
|
t.Errorf("FalseAction = %d, want 0", rep.FalseAction)
|
|
}
|
|
km := rep.ByRoute[RouteKnowledge]
|
|
if km.Precision != 0.25 {
|
|
t.Errorf("knowledge precision = %.3f, want 0.250", km.Precision)
|
|
}
|
|
if km.Recall != 1.0 {
|
|
t.Errorf("knowledge recall = %.3f, want 1.000", km.Recall)
|
|
}
|
|
t.Logf("eval report:\n%s", rep.String())
|
|
}
|
|
|
|
func TestShadowHarness(t *testing.T) {
|
|
model := SemanticRouterFunc(func(ctx context.Context, text string) (SemanticRouteDecision, error) {
|
|
if text == "выключи свет" {
|
|
return SemanticRouteDecision{Route: RouteAction, Confidence: 0.9}, nil
|
|
}
|
|
return SemanticRouteDecision{Route: RouteConversation, Confidence: 0.7}, nil
|
|
})
|
|
|
|
h := NewShadowHarness(model)
|
|
|
|
h.Observe(context.Background(), "выключи свет",
|
|
router.Decision{Intent: router.IntentAct}, false)
|
|
h.Observe(context.Background(), "привет",
|
|
router.Decision{Intent: router.IntentChat}, false)
|
|
h.Observe(context.Background(), "перезапусти докер",
|
|
router.Decision{Intent: router.IntentAct}, true)
|
|
|
|
rep := h.Summarize()
|
|
if rep.Total != 3 {
|
|
t.Errorf("Total = %d, want 3", rep.Total)
|
|
}
|
|
if rep.Agree != 2 {
|
|
t.Errorf("Agree = %d, want 2", rep.Agree)
|
|
}
|
|
if rep.Disagree != 1 {
|
|
t.Errorf("Disagree = %d, want 1", rep.Disagree)
|
|
}
|
|
if rep.FastPathTotal != 1 {
|
|
t.Errorf("FastPathTotal = %d, want 1", rep.FastPathTotal)
|
|
}
|
|
if rep.ResidualTotal != 2 {
|
|
t.Errorf("ResidualTotal = %d, want 2", rep.ResidualTotal)
|
|
}
|
|
t.Logf("shadow report:\n%s", rep.String())
|
|
}
|
|
|
|
func TestShadowHarnessNilModel(t *testing.T) {
|
|
h := NewShadowHarness(nil)
|
|
h.Observe(context.Background(), "привет",
|
|
router.Decision{Intent: router.IntentChat}, false)
|
|
rep := h.Summarize()
|
|
if rep.Total != 1 {
|
|
t.Errorf("Total = %d, want 1", rep.Total)
|
|
}
|
|
if rep.Agree != 0 {
|
|
t.Errorf("Agree = %d, want 0 (nil model → uncertain)", rep.Agree)
|
|
}
|
|
}
|