diff --git a/internal/router/eval/resolution_test.go b/internal/router/eval/resolution_test.go new file mode 100644 index 0000000..885ed29 --- /dev/null +++ b/internal/router/eval/resolution_test.go @@ -0,0 +1,220 @@ +package eval + +import ( + "context" + "fmt" + "testing" + + "github.com/kami/maven/internal/router" +) + +// resolutionRow — one cell in the resolution matrix. +type resolutionRow struct { + Producer router.RouteProducer + Method router.ActionResolutionMethod + Resolved int + Unresolved int + Fns map[string]int +} + +// printMatrix renders a resolution matrix from act outcomes. +func printMatrix(t *testing.T, label string, total int, matrix map[[2]string]*resolutionRow) { + t.Helper() + t.Logf("\n=== %s (%d act cases) ===", label, total) + t.Logf("%-15s %-25s %8s %8s %s", "producer", "method", "resolved", "unresolved", "fns") + keys := make([][2]string, 0, len(matrix)) + for k := range matrix { + keys = append(keys, k) + } + for i := 0; i < len(keys); i++ { + for j := i + 1; j < len(keys); j++ { + if keys[i][0]+keys[i][1] > keys[j][0]+keys[j][1] { + keys[i], keys[j] = keys[j], keys[i] + } + } + } + for _, k := range keys { + row := matrix[k] + fnList := "" + for fn, n := range row.Fns { + if fnList != "" { + fnList += ", " + } + fnList += fmt.Sprintf("%s×%d", fn, n) + } + t.Logf("%-15s %-25s %8d %8d %s", row.Producer, row.Method, row.Resolved, row.Unresolved, fnList) + } +} + +// TestResolutionMethodMatrix — diagnostic: runs both fixtures through the +// baseline router and reports which component selected the exact function. +func TestResolutionMethodMatrix(t *testing.T) { + emb := router.NewHashEmbedder(1024) + m := router.DefaultActMatcher{Fns: actFns} + ctx := context.Background() + + // --- routing fixture --- + rf, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + rfNow, err := rf.Now() + if err != nil { + t.Fatalf("Now: %v", err) + } + r := newBaselineRouter(t, emb, nil) + + type actOutcome struct { + Producer router.RouteProducer + Method router.ActionResolutionMethod + Fn string + Resolved bool + } + + // routing fixture + { + matrix := map[[2]string]*resolutionRow{} + total := 0 + resolved := 0 + counts := [5]int{} // fixed, matcher, raw, llm, fallback + for _, c := range rf.Cases { + d, err := r.Route(ctx, c.Utterance, rfNow) + if err != nil || d.Intent != router.IntentAct { + continue + } + total++ + candidate := router.ResolveActionCandidate(d, m) + key := [2]string{string(d.Producer), string(candidate.ResolvedBy)} + row, ok := matrix[key] + if !ok { + row = &resolutionRow{Producer: d.Producer, Method: candidate.ResolvedBy, Fns: map[string]int{}} + matrix[key] = row + } + if candidate.ActionResolved() { + resolved++ + row.Resolved++ + row.Fns[candidate.Fn]++ + switch candidate.ResolvedBy { + case router.ActionResolutionGrammarFixed: + counts[0]++ + case router.ActionResolutionGrammarMatcher: + counts[1]++ + case router.ActionResolutionExtractorRaw: + counts[2]++ + case router.ActionResolutionExtractorLLMText: + counts[3]++ + case router.ActionResolutionFallbackMatcher: + counts[4]++ + } + } else { + row.Unresolved++ + } + } + printMatrix(t, "Routing fixture: IntentAct resolution", total, matrix) + t.Logf("resolved: %d, unresolved: %d", resolved, total-resolved) + t.Logf("grammar_fixed=%d grammar_matcher=%d extractor_raw=%d extractor_llm_text=%d fallback_matcher=%d", + counts[0], counts[1], counts[2], counts[3], counts[4]) + } + + // ecosystem fixture + { + ef, err := LoadReach() + if err != nil { + t.Fatalf("LoadReach: %v", err) + } + efNow, err := ef.Now() + if err != nil { + t.Fatalf("Now: %v", err) + } + r2 := newBaselineRouter(t, emb, nil) + matrix := map[[2]string]*resolutionRow{} + total := 0 + resolved := 0 + counts := [5]int{} + for _, c := range ef.Cases { + d, err := r2.Route(ctx, c.Utterance, efNow) + if err != nil || d.Intent != router.IntentAct { + continue + } + total++ + candidate := router.ResolveActionCandidate(d, m) + key := [2]string{string(d.Producer), string(candidate.ResolvedBy)} + row, ok := matrix[key] + if !ok { + row = &resolutionRow{Producer: d.Producer, Method: candidate.ResolvedBy, Fns: map[string]int{}} + matrix[key] = row + } + if candidate.ActionResolved() { + resolved++ + row.Resolved++ + row.Fns[candidate.Fn]++ + switch candidate.ResolvedBy { + case router.ActionResolutionGrammarFixed: + counts[0]++ + case router.ActionResolutionGrammarMatcher: + counts[1]++ + case router.ActionResolutionExtractorRaw: + counts[2]++ + case router.ActionResolutionExtractorLLMText: + counts[3]++ + case router.ActionResolutionFallbackMatcher: + counts[4]++ + } + } else { + row.Unresolved++ + } + } + printMatrix(t, "Ecosystem fixture: IntentAct resolution", total, matrix) + t.Logf("resolved: %d, unresolved: %d", resolved, total-resolved) + t.Logf("grammar_fixed=%d grammar_matcher=%d extractor_raw=%d extractor_llm_text=%d fallback_matcher=%d", + counts[0], counts[1], counts[2], counts[3], counts[4]) + } +} + +// TestShadowMatcherComparison — diagnostic: for each IntentAct case with +// HasFn=true, invoke the fallback matcher on the same text and compare. +func TestShadowMatcherComparison(t *testing.T) { + f, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + now, err := f.Now() + if err != nil { + t.Fatalf("Now: %v", err) + } + emb := router.NewHashEmbedder(1024) + r := newBaselineRouter(t, emb, nil) + m := router.DefaultActMatcher{Fns: actFns} + ctx := context.Background() + + same, different, routeOnly, matcherOnly, missBoth, total := 0, 0, 0, 0, 0, 0 + for _, c := range f.Cases { + d, err := r.Route(ctx, c.Utterance, now) + if err != nil || d.Intent != router.IntentAct { + continue + } + total++ + if d.Slots.HasFn { + mFn, _, mOk := m.Match(d.Slots.Text) + if !mOk { + routeOnly++ + t.Logf(" %s %q: route=%s, matcher=miss", c.ID, c.Utterance, d.Slots.Fn) + } else if mFn == d.Slots.Fn { + same++ + } else { + different++ + t.Logf(" %s %q: route=%s, matcher=%s", c.ID, c.Utterance, d.Slots.Fn, mFn) + } + } else { + _, _, mOk := m.Match(d.Slots.Text) + if !mOk { + missBoth++ + } else { + matcherOnly++ + } + } + } + t.Logf("\n=== Shadow matcher comparison (%d act cases) ===", total) + t.Logf("same=%d different=%d routeOnly=%d matcherOnly=%d bothMiss=%d", + same, different, routeOnly, matcherOnly, missBoth) +}