package eval import ( "context" "fmt" "testing" "github.com/kami/maven/internal/router" ) // resolutionRow — one cell in the resolution matrix. type resolutionRow struct { Producer router.RouteProducer Method router.ActionResolutionMethod Resolved int Unresolved int Fns map[string]int } // printMatrix renders a resolution matrix from act outcomes. func printMatrix(t *testing.T, label string, total int, matrix map[[2]string]*resolutionRow) { t.Helper() t.Logf("\n=== %s (%d act cases) ===", label, total) t.Logf("%-15s %-25s %8s %8s %s", "producer", "method", "resolved", "unresolved", "fns") keys := make([][2]string, 0, len(matrix)) for k := range matrix { keys = append(keys, k) } for i := 0; i < len(keys); i++ { for j := i + 1; j < len(keys); j++ { if keys[i][0]+keys[i][1] > keys[j][0]+keys[j][1] { keys[i], keys[j] = keys[j], keys[i] } } } for _, k := range keys { row := matrix[k] fnList := "" for fn, n := range row.Fns { if fnList != "" { fnList += ", " } fnList += fmt.Sprintf("%s×%d", fn, n) } t.Logf("%-15s %-25s %8d %8d %s", row.Producer, row.Method, row.Resolved, row.Unresolved, fnList) } } // TestResolutionMethodMatrix — diagnostic: runs both fixtures through the // baseline router and reports which component selected the exact function. func TestResolutionMethodMatrix(t *testing.T) { emb := router.NewHashEmbedder(1024) m := router.DefaultActMatcher{Fns: actFns} ctx := context.Background() // --- routing fixture --- rf, err := Load() if err != nil { t.Fatalf("Load: %v", err) } rfNow, err := rf.Now() if err != nil { t.Fatalf("Now: %v", err) } r := newBaselineRouter(t, emb, nil) type actOutcome struct { Producer router.RouteProducer Method router.ActionResolutionMethod Fn string Resolved bool } // routing fixture { matrix := map[[2]string]*resolutionRow{} total := 0 resolved := 0 counts := [5]int{} // fixed, matcher, raw, llm, fallback for _, c := range rf.Cases { d, err := r.Route(ctx, router.NormalizedInput{Text: c.Utterance}, rfNow) if err != nil || d.Intent != router.IntentAct { continue } total++ candidate := router.ResolveActionCandidate(d, m) key := [2]string{string(d.Producer), string(candidate.ResolvedBy)} row, ok := matrix[key] if !ok { row = &resolutionRow{Producer: d.Producer, Method: candidate.ResolvedBy, Fns: map[string]int{}} matrix[key] = row } if candidate.ActionResolved() { resolved++ row.Resolved++ row.Fns[candidate.Fn]++ switch candidate.ResolvedBy { case router.ActionResolutionGrammarFixed: counts[0]++ case router.ActionResolutionGrammarMatcher: counts[1]++ case router.ActionResolutionExtractorRaw: counts[2]++ case router.ActionResolutionExtractorLLMText: counts[3]++ case router.ActionResolutionFallbackMatcher: counts[4]++ } } else { row.Unresolved++ } } printMatrix(t, "Routing fixture: IntentAct resolution", total, matrix) t.Logf("resolved: %d, unresolved: %d", resolved, total-resolved) t.Logf("grammar_fixed=%d grammar_matcher=%d extractor_raw=%d extractor_llm_text=%d fallback_matcher=%d", counts[0], counts[1], counts[2], counts[3], counts[4]) } // ecosystem fixture { ef, err := LoadReach() if err != nil { t.Fatalf("LoadReach: %v", err) } efNow, err := ef.Now() if err != nil { t.Fatalf("Now: %v", err) } r2 := newBaselineRouter(t, emb, nil) matrix := map[[2]string]*resolutionRow{} total := 0 resolved := 0 counts := [5]int{} for _, c := range ef.Cases { d, err := r2.Route(ctx, router.NormalizedInput{Text: c.Utterance}, efNow) if err != nil || d.Intent != router.IntentAct { continue } total++ candidate := router.ResolveActionCandidate(d, m) key := [2]string{string(d.Producer), string(candidate.ResolvedBy)} row, ok := matrix[key] if !ok { row = &resolutionRow{Producer: d.Producer, Method: candidate.ResolvedBy, Fns: map[string]int{}} matrix[key] = row } if candidate.ActionResolved() { resolved++ row.Resolved++ row.Fns[candidate.Fn]++ switch candidate.ResolvedBy { case router.ActionResolutionGrammarFixed: counts[0]++ case router.ActionResolutionGrammarMatcher: counts[1]++ case router.ActionResolutionExtractorRaw: counts[2]++ case router.ActionResolutionExtractorLLMText: counts[3]++ case router.ActionResolutionFallbackMatcher: counts[4]++ } } else { row.Unresolved++ } } printMatrix(t, "Ecosystem fixture: IntentAct resolution", total, matrix) t.Logf("resolved: %d, unresolved: %d", resolved, total-resolved) t.Logf("grammar_fixed=%d grammar_matcher=%d extractor_raw=%d extractor_llm_text=%d fallback_matcher=%d", counts[0], counts[1], counts[2], counts[3], counts[4]) } } // TestShadowMatcherComparison — diagnostic: for each IntentAct case with // HasFn=true, invoke the fallback matcher on the same text and compare. func TestShadowMatcherComparison(t *testing.T) { f, err := Load() if err != nil { t.Fatalf("Load: %v", err) } now, err := f.Now() if err != nil { t.Fatalf("Now: %v", err) } emb := router.NewHashEmbedder(1024) r := newBaselineRouter(t, emb, nil) m := router.DefaultActMatcher{Fns: actFns} ctx := context.Background() same, different, routeOnly, matcherOnly, missBoth, total := 0, 0, 0, 0, 0, 0 for _, c := range f.Cases { d, err := r.Route(ctx, router.NormalizedInput{Text: c.Utterance}, now) if err != nil || d.Intent != router.IntentAct { continue } total++ if d.Slots.HasFn { mFn, _, mOk := m.Match(d.Slots.Text) if !mOk { routeOnly++ t.Logf(" %s %q: route=%s, matcher=miss", c.ID, c.Utterance, d.Slots.Fn) } else if mFn == d.Slots.Fn { same++ } else { different++ t.Logf(" %s %q: route=%s, matcher=%s", c.ID, c.Utterance, d.Slots.Fn, mFn) } } else { _, _, mOk := m.Match(d.Slots.Text) if !mOk { missBoth++ } else { matcherOnly++ } } } t.Logf("\n=== Shadow matcher comparison (%d act cases) ===", total) t.Logf("same=%d different=%d routeOnly=%d matcherOnly=%d bothMiss=%d", same, different, routeOnly, matcherOnly, missBoth) }