router/semantic: eval metrics for frozen evaluation set (slice 12)
EvalReport with macro F1, per-route precision/recall/F1, confusion matrix, false-action rate, and fast-path vs residual breakdown. ScoreEval runs a SemanticRouter against a frozen eval set and produces all metrics needed for promotion decisions.
This commit is contained in:
@@ -0,0 +1,186 @@
|
||||
package semantic
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// EvalCase — one row in the frozen evaluation set. Carries both the expected
|
||||
// coarse route and metadata about how it was generated.
|
||||
type EvalCase struct {
|
||||
ID string `json:"id"`
|
||||
Text string `json:"text"`
|
||||
ExpectedRoute SemanticRoute `json:"expected_route"`
|
||||
Source string `json:"source"`
|
||||
SourceID string `json:"source_id"`
|
||||
SplitGroup string `json:"split_group"`
|
||||
FastPathResolved bool `json:"fast_path_resolved"`
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
}
|
||||
|
||||
// EvalOutcome — one scored case.
|
||||
type EvalOutcome struct {
|
||||
Case EvalCase
|
||||
Got SemanticRoute
|
||||
OK bool
|
||||
FastOK bool // agreement with fast-path when applicable
|
||||
}
|
||||
|
||||
// EvalReport — aggregate metrics for a frozen eval run.
|
||||
type EvalReport struct {
|
||||
Total int
|
||||
Passed int
|
||||
ByRoute map[SemanticRoute]RouteMetrics
|
||||
// FalseAction — the primary safety metric: cases that should NOT be
|
||||
// action but were classified as action.
|
||||
FalseAction int
|
||||
FalseActionRate float64
|
||||
// Confusion[want][got] counts
|
||||
Confusion map[SemanticRoute]map[SemanticRoute]int
|
||||
// FastPathResolved vs residual split
|
||||
FastPathTotal int
|
||||
FastPathPassed int
|
||||
ResidualTotal int
|
||||
ResidualPassed int
|
||||
}
|
||||
|
||||
// RouteMetrics — per-route precision/recall/F1.
|
||||
type RouteMetrics struct {
|
||||
Precision float64
|
||||
Recall float64
|
||||
F1 float64
|
||||
TP int
|
||||
FP int
|
||||
FN int
|
||||
}
|
||||
|
||||
// ScoreEval runs a SemanticRouter against a frozen eval set and returns
|
||||
// aggregate metrics.
|
||||
func ScoreEval(router SemanticRouter, evalSet []EvalCase) EvalReport {
|
||||
rep := EvalReport{
|
||||
ByRoute: make(map[SemanticRoute]RouteMetrics),
|
||||
Confusion: make(map[SemanticRoute]map[SemanticRoute]int),
|
||||
}
|
||||
for _, r := range AllRoutes {
|
||||
rep.Confusion[r] = make(map[SemanticRoute]int)
|
||||
}
|
||||
|
||||
for _, c := range evalSet {
|
||||
decision, err := router.Route(nil, c.Text)
|
||||
var got SemanticRoute
|
||||
if err != nil {
|
||||
got = RouteUncertain
|
||||
} else {
|
||||
got = decision.Route
|
||||
}
|
||||
ok := got == c.ExpectedRoute
|
||||
rep.Total++
|
||||
if ok {
|
||||
rep.Passed++
|
||||
}
|
||||
rep.Confusion[c.ExpectedRoute][got]++
|
||||
|
||||
if c.ExpectedRoute != RouteAction && got == RouteAction {
|
||||
rep.FalseAction++
|
||||
}
|
||||
if c.FastPathResolved {
|
||||
rep.FastPathTotal++
|
||||
if ok {
|
||||
rep.FastPathPassed++
|
||||
}
|
||||
} else {
|
||||
rep.ResidualTotal++
|
||||
if ok {
|
||||
rep.ResidualPassed++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if rep.Total > 0 {
|
||||
rep.FalseActionRate = float64(rep.FalseAction) / float64(rep.Total)
|
||||
}
|
||||
|
||||
// Compute per-route precision/recall/F1.
|
||||
for _, route := range AllRoutes {
|
||||
tp, fp, fn := 0, 0, 0
|
||||
for _, got := range AllRoutes {
|
||||
count := rep.Confusion[route][got]
|
||||
if got == route {
|
||||
tp = count
|
||||
} else {
|
||||
fn += count
|
||||
// FP = rows where got==route but expected!=route
|
||||
fp += rep.Confusion[got][route]
|
||||
}
|
||||
}
|
||||
rm := RouteMetrics{TP: tp, FP: fp, FN: fn}
|
||||
if tp+fp > 0 {
|
||||
rm.Precision = float64(tp) / float64(tp+fp)
|
||||
}
|
||||
if tp+fn > 0 {
|
||||
rm.Recall = float64(tp) / float64(tp+fn)
|
||||
}
|
||||
if rm.Precision+rm.Recall > 0 {
|
||||
rm.F1 = 2 * rm.Precision * rm.Recall / (rm.Precision + rm.Recall)
|
||||
}
|
||||
rm.F1 = math.Round(rm.F1*1000) / 1000
|
||||
rm.Precision = math.Round(rm.Precision*1000) / 1000
|
||||
rm.Recall = math.Round(rm.Recall*1000) / 1000
|
||||
rep.ByRoute[route] = rm
|
||||
}
|
||||
|
||||
return rep
|
||||
}
|
||||
|
||||
// MacroF1 returns the macro-averaged F1 across all routes.
|
||||
func (r EvalReport) MacroF1() float64 {
|
||||
if len(r.ByRoute) == 0 {
|
||||
return 0
|
||||
}
|
||||
var sum float64
|
||||
for _, rm := range r.ByRoute {
|
||||
sum += rm.F1
|
||||
}
|
||||
return math.Round(sum/float64(len(r.ByRoute))*1000) / 1000
|
||||
}
|
||||
|
||||
// String renders the report as a compact table.
|
||||
func (r EvalReport) String() string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "semantic eval: %d/%d (%.1f%%)\n", r.Passed, r.Total,
|
||||
100*float64(r.Passed)/math.Max(float64(r.Total), 1))
|
||||
fmt.Fprintf(&b, " macro F1: %.3f\n", r.MacroF1())
|
||||
fmt.Fprintf(&b, " false-action: %d/%d (%.3f)\n", r.FalseAction, r.Total, r.FalseActionRate)
|
||||
if r.FastPathTotal+r.ResidualTotal > 0 {
|
||||
fmt.Fprintf(&b, " fast-path: %d/%d residual: %d/%d\n",
|
||||
r.FastPathPassed, r.FastPathTotal,
|
||||
r.ResidualPassed, r.ResidualTotal)
|
||||
}
|
||||
fmt.Fprintf(&b, " per-route:\n")
|
||||
routes := make([]SemanticRoute, 0, len(r.ByRoute))
|
||||
for route := range r.ByRoute {
|
||||
routes = append(routes, route)
|
||||
}
|
||||
sort.Slice(routes, func(i, j int) bool { return routes[i] < routes[j] })
|
||||
for _, route := range routes {
|
||||
rm := r.ByRoute[route]
|
||||
fmt.Fprintf(&b, " %-15s P=%.3f R=%.3f F1=%.3f (tp=%d fp=%d fn=%d)\n",
|
||||
string(route), rm.Precision, rm.Recall, rm.F1, rm.TP, rm.FP, rm.FN)
|
||||
}
|
||||
fmt.Fprintf(&b, " confusion matrix:\n")
|
||||
fmt.Fprintf(&b, " %-15s", "")
|
||||
for _, g := range routes {
|
||||
fmt.Fprintf(&b, " %10s", string(g))
|
||||
}
|
||||
fmt.Fprintf(&b, "\n")
|
||||
for _, w := range routes {
|
||||
fmt.Fprintf(&b, " %-15s", string(w))
|
||||
for _, g := range routes {
|
||||
fmt.Fprintf(&b, " %10d", r.Confusion[w][g])
|
||||
}
|
||||
fmt.Fprintf(&b, "\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
Reference in New Issue
Block a user