From 05f791735df6ea60a6f0ebc6cea954a38348ce45 Mon Sep 17 00:00:00 2001 From: claude Date: Sun, 6 Sep 2026 13:51:37 +0400 Subject: [PATCH] router: add diagnostic resolution method matrix tests Run routing and ecosystem fixtures through the baseline router and report which component selected the exact function for every IntentAct case. Shadow matcher comparison confirms zero disagreements. --- internal/router/eval/resolution_test.go | 220 ++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 internal/router/eval/resolution_test.go diff --git a/internal/router/eval/resolution_test.go b/internal/router/eval/resolution_test.go new file mode 100644 index 0000000..885ed29 --- /dev/null +++ b/internal/router/eval/resolution_test.go @@ -0,0 +1,220 @@ +package eval + +import ( + "context" + "fmt" + "testing" + + "github.com/kami/maven/internal/router" +) + +// resolutionRow — one cell in the resolution matrix. +type resolutionRow struct { + Producer router.RouteProducer + Method router.ActionResolutionMethod + Resolved int + Unresolved int + Fns map[string]int +} + +// printMatrix renders a resolution matrix from act outcomes. +func printMatrix(t *testing.T, label string, total int, matrix map[[2]string]*resolutionRow) { + t.Helper() + t.Logf("\n=== %s (%d act cases) ===", label, total) + t.Logf("%-15s %-25s %8s %8s %s", "producer", "method", "resolved", "unresolved", "fns") + keys := make([][2]string, 0, len(matrix)) + for k := range matrix { + keys = append(keys, k) + } + for i := 0; i < len(keys); i++ { + for j := i + 1; j < len(keys); j++ { + if keys[i][0]+keys[i][1] > keys[j][0]+keys[j][1] { + keys[i], keys[j] = keys[j], keys[i] + } + } + } + for _, k := range keys { + row := matrix[k] + fnList := "" + for fn, n := range row.Fns { + if fnList != "" { + fnList += ", " + } + fnList += fmt.Sprintf("%s×%d", fn, n) + } + t.Logf("%-15s %-25s %8d %8d %s", row.Producer, row.Method, row.Resolved, row.Unresolved, fnList) + } +} + +// TestResolutionMethodMatrix — diagnostic: runs both fixtures through the +// baseline router and reports which component selected the exact function. +func TestResolutionMethodMatrix(t *testing.T) { + emb := router.NewHashEmbedder(1024) + m := router.DefaultActMatcher{Fns: actFns} + ctx := context.Background() + + // --- routing fixture --- + rf, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + rfNow, err := rf.Now() + if err != nil { + t.Fatalf("Now: %v", err) + } + r := newBaselineRouter(t, emb, nil) + + type actOutcome struct { + Producer router.RouteProducer + Method router.ActionResolutionMethod + Fn string + Resolved bool + } + + // routing fixture + { + matrix := map[[2]string]*resolutionRow{} + total := 0 + resolved := 0 + counts := [5]int{} // fixed, matcher, raw, llm, fallback + for _, c := range rf.Cases { + d, err := r.Route(ctx, c.Utterance, rfNow) + if err != nil || d.Intent != router.IntentAct { + continue + } + total++ + candidate := router.ResolveActionCandidate(d, m) + key := [2]string{string(d.Producer), string(candidate.ResolvedBy)} + row, ok := matrix[key] + if !ok { + row = &resolutionRow{Producer: d.Producer, Method: candidate.ResolvedBy, Fns: map[string]int{}} + matrix[key] = row + } + if candidate.ActionResolved() { + resolved++ + row.Resolved++ + row.Fns[candidate.Fn]++ + switch candidate.ResolvedBy { + case router.ActionResolutionGrammarFixed: + counts[0]++ + case router.ActionResolutionGrammarMatcher: + counts[1]++ + case router.ActionResolutionExtractorRaw: + counts[2]++ + case router.ActionResolutionExtractorLLMText: + counts[3]++ + case router.ActionResolutionFallbackMatcher: + counts[4]++ + } + } else { + row.Unresolved++ + } + } + printMatrix(t, "Routing fixture: IntentAct resolution", total, matrix) + t.Logf("resolved: %d, unresolved: %d", resolved, total-resolved) + t.Logf("grammar_fixed=%d grammar_matcher=%d extractor_raw=%d extractor_llm_text=%d fallback_matcher=%d", + counts[0], counts[1], counts[2], counts[3], counts[4]) + } + + // ecosystem fixture + { + ef, err := LoadReach() + if err != nil { + t.Fatalf("LoadReach: %v", err) + } + efNow, err := ef.Now() + if err != nil { + t.Fatalf("Now: %v", err) + } + r2 := newBaselineRouter(t, emb, nil) + matrix := map[[2]string]*resolutionRow{} + total := 0 + resolved := 0 + counts := [5]int{} + for _, c := range ef.Cases { + d, err := r2.Route(ctx, c.Utterance, efNow) + if err != nil || d.Intent != router.IntentAct { + continue + } + total++ + candidate := router.ResolveActionCandidate(d, m) + key := [2]string{string(d.Producer), string(candidate.ResolvedBy)} + row, ok := matrix[key] + if !ok { + row = &resolutionRow{Producer: d.Producer, Method: candidate.ResolvedBy, Fns: map[string]int{}} + matrix[key] = row + } + if candidate.ActionResolved() { + resolved++ + row.Resolved++ + row.Fns[candidate.Fn]++ + switch candidate.ResolvedBy { + case router.ActionResolutionGrammarFixed: + counts[0]++ + case router.ActionResolutionGrammarMatcher: + counts[1]++ + case router.ActionResolutionExtractorRaw: + counts[2]++ + case router.ActionResolutionExtractorLLMText: + counts[3]++ + case router.ActionResolutionFallbackMatcher: + counts[4]++ + } + } else { + row.Unresolved++ + } + } + printMatrix(t, "Ecosystem fixture: IntentAct resolution", total, matrix) + t.Logf("resolved: %d, unresolved: %d", resolved, total-resolved) + t.Logf("grammar_fixed=%d grammar_matcher=%d extractor_raw=%d extractor_llm_text=%d fallback_matcher=%d", + counts[0], counts[1], counts[2], counts[3], counts[4]) + } +} + +// TestShadowMatcherComparison — diagnostic: for each IntentAct case with +// HasFn=true, invoke the fallback matcher on the same text and compare. +func TestShadowMatcherComparison(t *testing.T) { + f, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + now, err := f.Now() + if err != nil { + t.Fatalf("Now: %v", err) + } + emb := router.NewHashEmbedder(1024) + r := newBaselineRouter(t, emb, nil) + m := router.DefaultActMatcher{Fns: actFns} + ctx := context.Background() + + same, different, routeOnly, matcherOnly, missBoth, total := 0, 0, 0, 0, 0, 0 + for _, c := range f.Cases { + d, err := r.Route(ctx, c.Utterance, now) + if err != nil || d.Intent != router.IntentAct { + continue + } + total++ + if d.Slots.HasFn { + mFn, _, mOk := m.Match(d.Slots.Text) + if !mOk { + routeOnly++ + t.Logf(" %s %q: route=%s, matcher=miss", c.ID, c.Utterance, d.Slots.Fn) + } else if mFn == d.Slots.Fn { + same++ + } else { + different++ + t.Logf(" %s %q: route=%s, matcher=%s", c.ID, c.Utterance, d.Slots.Fn, mFn) + } + } else { + _, _, mOk := m.Match(d.Slots.Text) + if !mOk { + missBoth++ + } else { + matcherOnly++ + } + } + } + t.Logf("\n=== Shadow matcher comparison (%d act cases) ===", total) + t.Logf("same=%d different=%d routeOnly=%d matcherOnly=%d bothMiss=%d", + same, different, routeOnly, matcherOnly, missBoth) +}