router: add diagnostic resolution method matrix tests
Run routing and ecosystem fixtures through the baseline router and report which component selected the exact function for every IntentAct case. Shadow matcher comparison confirms zero disagreements.
This commit is contained in:
@@ -0,0 +1,220 @@
|
|||||||
|
package eval
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/kami/maven/internal/router"
|
||||||
|
)
|
||||||
|
|
||||||
|
// resolutionRow — one cell in the resolution matrix.
|
||||||
|
type resolutionRow struct {
|
||||||
|
Producer router.RouteProducer
|
||||||
|
Method router.ActionResolutionMethod
|
||||||
|
Resolved int
|
||||||
|
Unresolved int
|
||||||
|
Fns map[string]int
|
||||||
|
}
|
||||||
|
|
||||||
|
// printMatrix renders a resolution matrix from act outcomes.
|
||||||
|
func printMatrix(t *testing.T, label string, total int, matrix map[[2]string]*resolutionRow) {
|
||||||
|
t.Helper()
|
||||||
|
t.Logf("\n=== %s (%d act cases) ===", label, total)
|
||||||
|
t.Logf("%-15s %-25s %8s %8s %s", "producer", "method", "resolved", "unresolved", "fns")
|
||||||
|
keys := make([][2]string, 0, len(matrix))
|
||||||
|
for k := range matrix {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
for i := 0; i < len(keys); i++ {
|
||||||
|
for j := i + 1; j < len(keys); j++ {
|
||||||
|
if keys[i][0]+keys[i][1] > keys[j][0]+keys[j][1] {
|
||||||
|
keys[i], keys[j] = keys[j], keys[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, k := range keys {
|
||||||
|
row := matrix[k]
|
||||||
|
fnList := ""
|
||||||
|
for fn, n := range row.Fns {
|
||||||
|
if fnList != "" {
|
||||||
|
fnList += ", "
|
||||||
|
}
|
||||||
|
fnList += fmt.Sprintf("%s×%d", fn, n)
|
||||||
|
}
|
||||||
|
t.Logf("%-15s %-25s %8d %8d %s", row.Producer, row.Method, row.Resolved, row.Unresolved, fnList)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestResolutionMethodMatrix — diagnostic: runs both fixtures through the
|
||||||
|
// baseline router and reports which component selected the exact function.
|
||||||
|
func TestResolutionMethodMatrix(t *testing.T) {
|
||||||
|
emb := router.NewHashEmbedder(1024)
|
||||||
|
m := router.DefaultActMatcher{Fns: actFns}
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
// --- routing fixture ---
|
||||||
|
rf, err := Load()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load: %v", err)
|
||||||
|
}
|
||||||
|
rfNow, err := rf.Now()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Now: %v", err)
|
||||||
|
}
|
||||||
|
r := newBaselineRouter(t, emb, nil)
|
||||||
|
|
||||||
|
type actOutcome struct {
|
||||||
|
Producer router.RouteProducer
|
||||||
|
Method router.ActionResolutionMethod
|
||||||
|
Fn string
|
||||||
|
Resolved bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// routing fixture
|
||||||
|
{
|
||||||
|
matrix := map[[2]string]*resolutionRow{}
|
||||||
|
total := 0
|
||||||
|
resolved := 0
|
||||||
|
counts := [5]int{} // fixed, matcher, raw, llm, fallback
|
||||||
|
for _, c := range rf.Cases {
|
||||||
|
d, err := r.Route(ctx, c.Utterance, rfNow)
|
||||||
|
if err != nil || d.Intent != router.IntentAct {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
total++
|
||||||
|
candidate := router.ResolveActionCandidate(d, m)
|
||||||
|
key := [2]string{string(d.Producer), string(candidate.ResolvedBy)}
|
||||||
|
row, ok := matrix[key]
|
||||||
|
if !ok {
|
||||||
|
row = &resolutionRow{Producer: d.Producer, Method: candidate.ResolvedBy, Fns: map[string]int{}}
|
||||||
|
matrix[key] = row
|
||||||
|
}
|
||||||
|
if candidate.ActionResolved() {
|
||||||
|
resolved++
|
||||||
|
row.Resolved++
|
||||||
|
row.Fns[candidate.Fn]++
|
||||||
|
switch candidate.ResolvedBy {
|
||||||
|
case router.ActionResolutionGrammarFixed:
|
||||||
|
counts[0]++
|
||||||
|
case router.ActionResolutionGrammarMatcher:
|
||||||
|
counts[1]++
|
||||||
|
case router.ActionResolutionExtractorRaw:
|
||||||
|
counts[2]++
|
||||||
|
case router.ActionResolutionExtractorLLMText:
|
||||||
|
counts[3]++
|
||||||
|
case router.ActionResolutionFallbackMatcher:
|
||||||
|
counts[4]++
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
row.Unresolved++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
printMatrix(t, "Routing fixture: IntentAct resolution", total, matrix)
|
||||||
|
t.Logf("resolved: %d, unresolved: %d", resolved, total-resolved)
|
||||||
|
t.Logf("grammar_fixed=%d grammar_matcher=%d extractor_raw=%d extractor_llm_text=%d fallback_matcher=%d",
|
||||||
|
counts[0], counts[1], counts[2], counts[3], counts[4])
|
||||||
|
}
|
||||||
|
|
||||||
|
// ecosystem fixture
|
||||||
|
{
|
||||||
|
ef, err := LoadReach()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadReach: %v", err)
|
||||||
|
}
|
||||||
|
efNow, err := ef.Now()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Now: %v", err)
|
||||||
|
}
|
||||||
|
r2 := newBaselineRouter(t, emb, nil)
|
||||||
|
matrix := map[[2]string]*resolutionRow{}
|
||||||
|
total := 0
|
||||||
|
resolved := 0
|
||||||
|
counts := [5]int{}
|
||||||
|
for _, c := range ef.Cases {
|
||||||
|
d, err := r2.Route(ctx, c.Utterance, efNow)
|
||||||
|
if err != nil || d.Intent != router.IntentAct {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
total++
|
||||||
|
candidate := router.ResolveActionCandidate(d, m)
|
||||||
|
key := [2]string{string(d.Producer), string(candidate.ResolvedBy)}
|
||||||
|
row, ok := matrix[key]
|
||||||
|
if !ok {
|
||||||
|
row = &resolutionRow{Producer: d.Producer, Method: candidate.ResolvedBy, Fns: map[string]int{}}
|
||||||
|
matrix[key] = row
|
||||||
|
}
|
||||||
|
if candidate.ActionResolved() {
|
||||||
|
resolved++
|
||||||
|
row.Resolved++
|
||||||
|
row.Fns[candidate.Fn]++
|
||||||
|
switch candidate.ResolvedBy {
|
||||||
|
case router.ActionResolutionGrammarFixed:
|
||||||
|
counts[0]++
|
||||||
|
case router.ActionResolutionGrammarMatcher:
|
||||||
|
counts[1]++
|
||||||
|
case router.ActionResolutionExtractorRaw:
|
||||||
|
counts[2]++
|
||||||
|
case router.ActionResolutionExtractorLLMText:
|
||||||
|
counts[3]++
|
||||||
|
case router.ActionResolutionFallbackMatcher:
|
||||||
|
counts[4]++
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
row.Unresolved++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
printMatrix(t, "Ecosystem fixture: IntentAct resolution", total, matrix)
|
||||||
|
t.Logf("resolved: %d, unresolved: %d", resolved, total-resolved)
|
||||||
|
t.Logf("grammar_fixed=%d grammar_matcher=%d extractor_raw=%d extractor_llm_text=%d fallback_matcher=%d",
|
||||||
|
counts[0], counts[1], counts[2], counts[3], counts[4])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestShadowMatcherComparison — diagnostic: for each IntentAct case with
|
||||||
|
// HasFn=true, invoke the fallback matcher on the same text and compare.
|
||||||
|
func TestShadowMatcherComparison(t *testing.T) {
|
||||||
|
f, err := Load()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load: %v", err)
|
||||||
|
}
|
||||||
|
now, err := f.Now()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Now: %v", err)
|
||||||
|
}
|
||||||
|
emb := router.NewHashEmbedder(1024)
|
||||||
|
r := newBaselineRouter(t, emb, nil)
|
||||||
|
m := router.DefaultActMatcher{Fns: actFns}
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
same, different, routeOnly, matcherOnly, missBoth, total := 0, 0, 0, 0, 0, 0
|
||||||
|
for _, c := range f.Cases {
|
||||||
|
d, err := r.Route(ctx, c.Utterance, now)
|
||||||
|
if err != nil || d.Intent != router.IntentAct {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
total++
|
||||||
|
if d.Slots.HasFn {
|
||||||
|
mFn, _, mOk := m.Match(d.Slots.Text)
|
||||||
|
if !mOk {
|
||||||
|
routeOnly++
|
||||||
|
t.Logf(" %s %q: route=%s, matcher=miss", c.ID, c.Utterance, d.Slots.Fn)
|
||||||
|
} else if mFn == d.Slots.Fn {
|
||||||
|
same++
|
||||||
|
} else {
|
||||||
|
different++
|
||||||
|
t.Logf(" %s %q: route=%s, matcher=%s", c.ID, c.Utterance, d.Slots.Fn, mFn)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
_, _, mOk := m.Match(d.Slots.Text)
|
||||||
|
if !mOk {
|
||||||
|
missBoth++
|
||||||
|
} else {
|
||||||
|
matcherOnly++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Logf("\n=== Shadow matcher comparison (%d act cases) ===", total)
|
||||||
|
t.Logf("same=%d different=%d routeOnly=%d matcherOnly=%d bothMiss=%d",
|
||||||
|
same, different, routeOnly, matcherOnly, missBoth)
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user