Maven/internal/router/eval/resolution_test.go

package eval

import (
	"context"
	"fmt"
	"testing"

	"github.com/kami/maven/internal/router"
)

// resolutionRow — one cell in the resolution matrix.
type resolutionRow struct {
	Producer   router.RouteProducer
	Method     router.ActionResolutionMethod
	Resolved   int
	Unresolved int
	Fns        map[string]int
}

// printMatrix renders a resolution matrix from act outcomes.
func printMatrix(t *testing.T, label string, total int, matrix map[[2]string]*resolutionRow) {
	t.Helper()
	t.Logf("\n=== %s (%d act cases) ===", label, total)
	t.Logf("%-15s %-25s %8s %8s  %s", "producer", "method", "resolved", "unresolved", "fns")
	keys := make([][2]string, 0, len(matrix))
	for k := range matrix {
		keys = append(keys, k)
	}
	for i := 0; i < len(keys); i++ {
		for j := i + 1; j < len(keys); j++ {
			if keys[i][0]+keys[i][1] > keys[j][0]+keys[j][1] {
				keys[i], keys[j] = keys[j], keys[i]
			}
		}
	}
	for _, k := range keys {
		row := matrix[k]
		fnList := ""
		for fn, n := range row.Fns {
			if fnList != "" {
				fnList += ", "
			}
			fnList += fmt.Sprintf("%s×%d", fn, n)
		}
		t.Logf("%-15s %-25s %8d %8d  %s", row.Producer, row.Method, row.Resolved, row.Unresolved, fnList)
	}
}

// TestResolutionMethodMatrix — diagnostic: runs both fixtures through the
// baseline router and reports which component selected the exact function.
func TestResolutionMethodMatrix(t *testing.T) {
	emb := router.NewHashEmbedder(1024)
	m := router.DefaultActMatcher{Fns: actFns}
	ctx := context.Background()

	// --- routing fixture ---
	rf, err := Load()
	if err != nil {
		t.Fatalf("Load: %v", err)
	}
	rfNow, err := rf.Now()
	if err != nil {
		t.Fatalf("Now: %v", err)
	}
	r := newBaselineRouter(t, emb, nil)

	type actOutcome struct {
		Producer router.RouteProducer
		Method   router.ActionResolutionMethod
		Fn       string
		Resolved bool
	}

	// routing fixture
	{
		matrix := map[[2]string]*resolutionRow{}
		total := 0
		resolved := 0
	 counts := [5]int{} // fixed, matcher, raw, llm, fallback
		for _, c := range rf.Cases {
			d, err := r.Route(ctx, c.Utterance, rfNow)
			if err != nil || d.Intent != router.IntentAct {
				continue
			}
			total++
			candidate := router.ResolveActionCandidate(d, m)
			key := [2]string{string(d.Producer), string(candidate.ResolvedBy)}
			row, ok := matrix[key]
			if !ok {
				row = &resolutionRow{Producer: d.Producer, Method: candidate.ResolvedBy, Fns: map[string]int{}}
				matrix[key] = row
			}
			if candidate.ActionResolved() {
				resolved++
				row.Resolved++
				row.Fns[candidate.Fn]++
				switch candidate.ResolvedBy {
				case router.ActionResolutionGrammarFixed:
					counts[0]++
				case router.ActionResolutionGrammarMatcher:
					counts[1]++
				case router.ActionResolutionExtractorRaw:
					counts[2]++
				case router.ActionResolutionExtractorLLMText:
					counts[3]++
				case router.ActionResolutionFallbackMatcher:
					counts[4]++
				}
			} else {
				row.Unresolved++
			}
		}
		printMatrix(t, "Routing fixture: IntentAct resolution", total, matrix)
		t.Logf("resolved: %d, unresolved: %d", resolved, total-resolved)
		t.Logf("grammar_fixed=%d grammar_matcher=%d extractor_raw=%d extractor_llm_text=%d fallback_matcher=%d",
			counts[0], counts[1], counts[2], counts[3], counts[4])
	}

	// ecosystem fixture
	{
		ef, err := LoadReach()
		if err != nil {
			t.Fatalf("LoadReach: %v", err)
		}
		efNow, err := ef.Now()
		if err != nil {
			t.Fatalf("Now: %v", err)
		}
		r2 := newBaselineRouter(t, emb, nil)
		matrix := map[[2]string]*resolutionRow{}
		total := 0
		resolved := 0
		counts := [5]int{}
		for _, c := range ef.Cases {
			d, err := r2.Route(ctx, c.Utterance, efNow)
			if err != nil || d.Intent != router.IntentAct {
				continue
			}
			total++
			candidate := router.ResolveActionCandidate(d, m)
			key := [2]string{string(d.Producer), string(candidate.ResolvedBy)}
			row, ok := matrix[key]
			if !ok {
				row = &resolutionRow{Producer: d.Producer, Method: candidate.ResolvedBy, Fns: map[string]int{}}
				matrix[key] = row
			}
			if candidate.ActionResolved() {
				resolved++
				row.Resolved++
				row.Fns[candidate.Fn]++
				switch candidate.ResolvedBy {
				case router.ActionResolutionGrammarFixed:
					counts[0]++
				case router.ActionResolutionGrammarMatcher:
					counts[1]++
				case router.ActionResolutionExtractorRaw:
					counts[2]++
				case router.ActionResolutionExtractorLLMText:
					counts[3]++
				case router.ActionResolutionFallbackMatcher:
					counts[4]++
				}
			} else {
				row.Unresolved++
			}
		}
		printMatrix(t, "Ecosystem fixture: IntentAct resolution", total, matrix)
		t.Logf("resolved: %d, unresolved: %d", resolved, total-resolved)
		t.Logf("grammar_fixed=%d grammar_matcher=%d extractor_raw=%d extractor_llm_text=%d fallback_matcher=%d",
			counts[0], counts[1], counts[2], counts[3], counts[4])
	}
}

// TestShadowMatcherComparison — diagnostic: for each IntentAct case with
// HasFn=true, invoke the fallback matcher on the same text and compare.
func TestShadowMatcherComparison(t *testing.T) {
	f, err := Load()
	if err != nil {
		t.Fatalf("Load: %v", err)
	}
	now, err := f.Now()
	if err != nil {
		t.Fatalf("Now: %v", err)
	}
	emb := router.NewHashEmbedder(1024)
	r := newBaselineRouter(t, emb, nil)
	m := router.DefaultActMatcher{Fns: actFns}
	ctx := context.Background()

	same, different, routeOnly, matcherOnly, missBoth, total := 0, 0, 0, 0, 0, 0
	for _, c := range f.Cases {
		d, err := r.Route(ctx, c.Utterance, now)
		if err != nil || d.Intent != router.IntentAct {
			continue
		}
		total++
		if d.Slots.HasFn {
			mFn, _, mOk := m.Match(d.Slots.Text)
			if !mOk {
				routeOnly++
				t.Logf("  %s %q: route=%s, matcher=miss", c.ID, c.Utterance, d.Slots.Fn)
			} else if mFn == d.Slots.Fn {
				same++
			} else {
				different++
				t.Logf("  %s %q: route=%s, matcher=%s", c.ID, c.Utterance, d.Slots.Fn, mFn)
			}
		} else {
			_, _, mOk := m.Match(d.Slots.Text)
			if !mOk {
				missBoth++
			} else {
				matcherOnly++
			}
		}
	}
	t.Logf("\n=== Shadow matcher comparison (%d act cases) ===", total)
	t.Logf("same=%d different=%d routeOnly=%d matcherOnly=%d bothMiss=%d",
		same, different, routeOnly, matcherOnly, missBoth)
}