Maven/internal/router/semantic/eval.go

package semantic

import (
	"fmt"
	"math"
	"sort"
	"strings"
)

// EvalCase — one row in the frozen evaluation set. Carries both the expected
// coarse route and metadata about how it was generated.
type EvalCase struct {
	ID               string        `json:"id"`
	Text             string        `json:"text"`
	ExpectedRoute    SemanticRoute `json:"expected_route"`
	Source           string        `json:"source"`
	SourceID         string        `json:"source_id"`
	SplitGroup       string        `json:"split_group"`
	FastPathResolved bool          `json:"fast_path_resolved"`
	Tags             []string      `json:"tags,omitempty"`
}

// EvalOutcome — one scored case.
type EvalOutcome struct {
	Case   EvalCase
	Got    SemanticRoute
	OK     bool
	FastOK bool // agreement with fast-path when applicable
}

// EvalReport — aggregate metrics for a frozen eval run.
type EvalReport struct {
	Total   int
	Passed  int
	ByRoute map[SemanticRoute]RouteMetrics
	// FalseAction — the primary safety metric: cases that should NOT be
	// action but were classified as action.
	FalseAction     int
	FalseActionRate float64
	// Confusion[want][got] counts
	Confusion map[SemanticRoute]map[SemanticRoute]int
	// FastPathResolved vs residual split
	FastPathTotal  int
	FastPathPassed int
	ResidualTotal  int
	ResidualPassed int
}

// RouteMetrics — per-route precision/recall/F1.
type RouteMetrics struct {
	Precision float64
	Recall    float64
	F1        float64
	TP        int
	FP        int
	FN        int
}

// ScoreEval runs a SemanticRouter against a frozen eval set and returns
// aggregate metrics.
func ScoreEval(router SemanticRouter, evalSet []EvalCase) EvalReport {
	rep := EvalReport{
		ByRoute:  make(map[SemanticRoute]RouteMetrics),
		Confusion: make(map[SemanticRoute]map[SemanticRoute]int),
	}
	for _, r := range AllRoutes {
		rep.Confusion[r] = make(map[SemanticRoute]int)
	}

	for _, c := range evalSet {
		decision, err := router.Route(nil, c.Text)
		var got SemanticRoute
		if err != nil {
			got = RouteUncertain
		} else {
			got = decision.Route
		}
		ok := got == c.ExpectedRoute
		rep.Total++
		if ok {
			rep.Passed++
		}
		rep.Confusion[c.ExpectedRoute][got]++

		if c.ExpectedRoute != RouteAction && got == RouteAction {
			rep.FalseAction++
		}
		if c.FastPathResolved {
			rep.FastPathTotal++
			if ok {
				rep.FastPathPassed++
			}
		} else {
			rep.ResidualTotal++
			if ok {
				rep.ResidualPassed++
			}
		}
	}

	if rep.Total > 0 {
		rep.FalseActionRate = float64(rep.FalseAction) / float64(rep.Total)
	}

	// Compute per-route precision/recall/F1.
	for _, route := range AllRoutes {
		tp, fp, fn := 0, 0, 0
		for _, got := range AllRoutes {
			count := rep.Confusion[route][got]
			if got == route {
				tp = count
			} else {
				fn += count
				// FP = rows where got==route but expected!=route
				fp += rep.Confusion[got][route]
			}
		}
		rm := RouteMetrics{TP: tp, FP: fp, FN: fn}
		if tp+fp > 0 {
			rm.Precision = float64(tp) / float64(tp+fp)
		}
		if tp+fn > 0 {
			rm.Recall = float64(tp) / float64(tp+fn)
		}
		if rm.Precision+rm.Recall > 0 {
			rm.F1 = 2 * rm.Precision * rm.Recall / (rm.Precision + rm.Recall)
		}
		rm.F1 = math.Round(rm.F1*1000) / 1000
		rm.Precision = math.Round(rm.Precision*1000) / 1000
		rm.Recall = math.Round(rm.Recall*1000) / 1000
		rep.ByRoute[route] = rm
	}

	return rep
}

// MacroF1 returns the macro-averaged F1 across all routes.
func (r EvalReport) MacroF1() float64 {
	if len(r.ByRoute) == 0 {
		return 0
	}
	var sum float64
	for _, rm := range r.ByRoute {
		sum += rm.F1
	}
	return math.Round(sum/float64(len(r.ByRoute))*1000) / 1000
}

// String renders the report as a compact table.
func (r EvalReport) String() string {
	var b strings.Builder
	fmt.Fprintf(&b, "semantic eval: %d/%d (%.1f%%)\n", r.Passed, r.Total,
		100*float64(r.Passed)/math.Max(float64(r.Total), 1))
	fmt.Fprintf(&b, "  macro F1: %.3f\n", r.MacroF1())
	fmt.Fprintf(&b, "  false-action: %d/%d (%.3f)\n", r.FalseAction, r.Total, r.FalseActionRate)
	if r.FastPathTotal+r.ResidualTotal > 0 {
		fmt.Fprintf(&b, "  fast-path: %d/%d  residual: %d/%d\n",
			r.FastPathPassed, r.FastPathTotal,
			r.ResidualPassed, r.ResidualTotal)
	}
	fmt.Fprintf(&b, "  per-route:\n")
	routes := make([]SemanticRoute, 0, len(r.ByRoute))
	for route := range r.ByRoute {
		routes = append(routes, route)
	}
	sort.Slice(routes, func(i, j int) bool { return routes[i] < routes[j] })
	for _, route := range routes {
		rm := r.ByRoute[route]
		fmt.Fprintf(&b, "    %-15s P=%.3f R=%.3f F1=%.3f  (tp=%d fp=%d fn=%d)\n",
			string(route), rm.Precision, rm.Recall, rm.F1, rm.TP, rm.FP, rm.FN)
	}
	fmt.Fprintf(&b, "  confusion matrix:\n")
	fmt.Fprintf(&b, "    %-15s", "")
	for _, g := range routes {
		fmt.Fprintf(&b, " %10s", string(g))
	}
	fmt.Fprintf(&b, "\n")
	for _, w := range routes {
		fmt.Fprintf(&b, "    %-15s", string(w))
		for _, g := range routes {
			fmt.Fprintf(&b, " %10d", r.Confusion[w][g])
		}
		fmt.Fprintf(&b, "\n")
	}
	return b.String()
}