From 1b8ae3c3e5a2e3269b3066f896f3014ae2e27249 Mon Sep 17 00:00:00 2001 From: claude Date: Mon, 7 Sep 2026 02:08:51 +0400 Subject: [PATCH] router/semantic: baseline scoring and contrast family analysis (slice 13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ScoreLegacy runs the actual router cascade against the 136-example corpus and produces per-route precision/recall/F1, confusion matrix, false-action breakdown, and fast-path/residual/router-residual/pre-route consumption counts. Pre-route consumed cases (command-prohibition grammar matches at stage 0) are tracked separately — these never reach the general cascade and should not be scored by the learned router. ContrastFamilies splits the baseline report by transform tag (negation, question, reported_speech, quotation, hypothetical, capability_question) and reports per-family accuracy and false-action rate. LegacyReport.String() renders the full baseline report with confusion matrix and false-action case listing. --- internal/router/semantic/baseline.go | 241 +++++++++++++++++++++++++++ 1 file changed, 241 insertions(+) create mode 100644 internal/router/semantic/baseline.go diff --git a/internal/router/semantic/baseline.go b/internal/router/semantic/baseline.go new file mode 100644 index 0000000..82328cb --- /dev/null +++ b/internal/router/semantic/baseline.go @@ -0,0 +1,241 @@ +package semantic + +import ( + "context" + "fmt" + "math" + "sort" + "strings" + "time" + + "github.com/kami/maven/internal/router" +) + +// ScoreLegacy runs the actual router against the corpus and produces the +// baseline report. The now parameter is the reference clock for relative +// time expressions. +func ScoreLegacy(ctx context.Context, r LegacyRouter, exs []RouteExample, now time.Time) LegacyReport { + stats := CorpusStatsFrom(exs) + rep := LegacyReport{ + Stats: stats, + Total: len(exs), + ByRoute: make(map[SemanticRoute]RouteMetrics), + Confusion: make(map[SemanticRoute]map[SemanticRoute]int), + } + for _, rt := range AllRoutes { + rep.Confusion[rt] = make(map[SemanticRoute]int) + } + + for _, e := range exs { + input := router.NormalizedInput{Text: e.Text} + d, err := r.Route(ctx, input, now) + var predicted SemanticRoute + var fpHit bool + var preroute bool + if err != nil { + predicted = RouteUncertain + } else { + // Determine if fast-path resolved this. + fpHit = d.Stage == 0 && d.Producer == router.RouteProducerGrammar + predicted = IntentToRoute(d.Intent) + // Pre-route consumption: command-prohibition grammar emits + // IntentAct with Fn=prohibited_act at stage 0. These cases + // never reach the general cascade. + preroute = d.Stage == 0 && d.Slots.Fn == "prohibited_act" + } + agree := predicted == e.Route + c := LegacyCase{ + Example: e, + Decision: d, + Predicted: predicted, + Agree: agree, + FastPathHit: fpHit, + PrerouteConsumed: preroute, + Error: err, + } + rep.Cases = append(rep.Cases, c) + + rep.Confusion[e.Route][predicted]++ + if agree { + rep.Passed++ + } + if e.Route != RouteAction && predicted == RouteAction { + rep.FalseAction++ + rep.FalseActionCases = append(rep.FalseActionCases, c) + } + if e.FastPathResolved || fpHit { + if preroute { + // Command-prohibition grammar: consumed before cascade. + rep.PreRouteTotal++ + if agree { + rep.PreRoutePassed++ + } + } else { + rep.FastPathTotal++ + if agree { + rep.FastPathPassed++ + } + } + } else { + rep.ResidualTotal++ + if agree { + rep.ResidualPassed++ + } + // All residual cases are router-residual (the learned + // router would see all of them). + rep.RouterResidualTotal++ + if agree { + rep.RouterResidualPassed++ + } + } + } + + if rep.Total > 0 { + rep.FalseActionRate = float64(rep.FalseAction) / float64(rep.Total) + } + + // Per-route P/R/F1. + for _, route := range AllRoutes { + tp, fp, fn := 0, 0, 0 + for _, got := range AllRoutes { + count := rep.Confusion[route][got] + if got == route { + tp = count + } else { + fn += count + fp += rep.Confusion[got][route] + } + } + rm := RouteMetrics{TP: tp, FP: fp, FN: fn} + if tp+fp > 0 { + rm.Precision = float64(tp) / float64(tp+fp) + } + if tp+fn > 0 { + rm.Recall = float64(tp) / float64(tp+fn) + } + if rm.Precision+rm.Recall > 0 { + rm.F1 = 2 * rm.Precision * rm.Recall / (rm.Precision + rm.Recall) + } + rm.F1 = math.Round(rm.F1*1000) / 1000 + rm.Precision = math.Round(rm.Precision*1000) / 1000 + rm.Recall = math.Round(rm.Recall*1000) / 1000 + rep.ByRoute[route] = rm + } + + return rep +} + +// MacroF1 returns the macro-averaged F1 across all routes. +func (r LegacyReport) MacroF1() float64 { + if len(r.ByRoute) == 0 { + return 0 + } + var sum float64 + for _, rm := range r.ByRoute { + sum += rm.F1 + } + return math.Round(sum/float64(len(r.ByRoute))*1000) / 1000 +} + +// String renders the legacy baseline report. +func (r LegacyReport) String() string { + var b strings.Builder + fmt.Fprintf(&b, "legacy baseline: %d/%d (%.1f%%)\n", r.Passed, r.Total, + 100*float64(r.Passed)/math.Max(float64(r.Total), 1)) + fmt.Fprintf(&b, " macro F1: %.3f\n", r.MacroF1()) + fmt.Fprintf(&b, " false-action: %d/%d (%.3f)\n", r.FalseAction, r.Total, r.FalseActionRate) + fmt.Fprintf(&b, " fast-path: %d/%d residual: %d/%d\n", + r.FastPathPassed, r.FastPathTotal, + r.ResidualPassed, r.ResidualTotal) + if r.RouterResidualTotal > 0 { + fmt.Fprintf(&b, " router-residual: %d/%d (pre-route consumed: %d)\n", + r.RouterResidualPassed, r.RouterResidualTotal, r.PreRouteTotal) + } + fmt.Fprintf(&b, " per-route:\n") + routes := make([]SemanticRoute, 0, len(r.ByRoute)) + for route := range r.ByRoute { + routes = append(routes, route) + } + sort.Slice(routes, func(i, j int) bool { return routes[i] < routes[j] }) + for _, route := range routes { + rm := r.ByRoute[route] + fmt.Fprintf(&b, " %-15s P=%.3f R=%.3f F1=%.3f (tp=%d fp=%d fn=%d)\n", + string(route), rm.Precision, rm.Recall, rm.F1, rm.TP, rm.FP, rm.FN) + } + fmt.Fprintf(&b, " confusion matrix:\n") + fmt.Fprintf(&b, " %-15s", "") + for _, g := range routes { + fmt.Fprintf(&b, " %12s", string(g)) + } + fmt.Fprintf(&b, "\n") + for _, w := range routes { + fmt.Fprintf(&b, " %-15s", string(w)) + for _, g := range routes { + fmt.Fprintf(&b, " %12d", r.Confusion[w][g]) + } + fmt.Fprintf(&b, "\n") + } + if len(r.FalseActionCases) > 0 { + fmt.Fprintf(&b, " false-action cases:\n") + for _, c := range r.FalseActionCases { + decided := "(error)" + if c.Error == nil { + decided = fmt.Sprintf("%s (%.3f)", c.Decision.Intent, c.Decision.Confidence) + } + fmt.Fprintf(&b, " %s %q: expected %s, got action (decided %s)\n", + c.Example.SourceID, c.Example.Text, c.Example.Route, decided) + } + } + return b.String() +} + +// ContrastFamilies splits the corpus rows by their transform tag and reports +// per-family performance against the legacy router. +func ContrastFamilies(rep LegacyReport) []ContrastFamilyReport { + // Collect transform tags → cases. + tagCases := map[string][]LegacyCase{} + for _, c := range rep.Cases { + for _, tag := range c.Example.Tags { + switch tag { + case "negation", "question", "reported_speech", "quotation", + "hypothetical", "capability_question": + tagCases[tag] = append(tagCases[tag], c) + } + } + } + + var reports []ContrastFamilyReport + for _, tag := range []string{ + "negation", "question", "reported_speech", "quotation", + "hypothetical", "capability_question", + } { + cases := tagCases[tag] + if len(cases) == 0 { + continue + } + cr := ContrastFamilyReport{Transform: tag, Examples: len(cases), Cases: cases} + for _, c := range cases { + if c.Agree { + cr.Correct++ + } + if c.Example.Route != RouteAction && c.Predicted == RouteAction { + cr.FalseAction++ + } else if !c.Agree { + cr.OtherErrors++ + } + } + reports = append(reports, cr) + } + return reports +} + +// String renders the contrast family report. +func ContrastFamilyReportString(r []ContrastFamilyReport) string { + var b strings.Builder + fmt.Fprintf(&b, "%-20s %8s %8s %8s %8s\n", "transform", "examples", "correct", "false-act", "other") + for _, cr := range r { + fmt.Fprintf(&b, "%-20s %8d %8d %8d %8d\n", + cr.Transform, cr.Examples, cr.Correct, cr.FalseAction, cr.OtherErrors) + } + return b.String() +}