router/semantic: slice 23 corpus fast-path reconciliation — DeriveFastPath over the real router replaces the regex mirror, factory/merge validated, corpus rebuilt (dataset_hash unchanged), drift diagnostic and rerun tooling
This commit is contained in:
@@ -0,0 +1,243 @@
|
||||
// slice23 — fast-path metadata reconciliation diagnostic.
|
||||
//
|
||||
// Classifies every disagreement between the corpus's stored fast_path_resolved
|
||||
// flag and what the production fast path derives today (TryFastPath over the
|
||||
// stage-0 grammars with the experiment's act allowlist). Outputs a JSON
|
||||
// decomposition and a console summary for docs/evals reports.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// go run ./cmd/semantic-router-experiment/slice23/ -out /tmp/mvn-s23/drift.json
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/router/semantic"
|
||||
)
|
||||
|
||||
type conflict struct {
|
||||
Text string `json:"text"`
|
||||
SourceID string `json:"source_id"`
|
||||
Route string `json:"route"`
|
||||
Source string `json:"source"`
|
||||
Group string `json:"split_group"`
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
Dev bool `json:"dev"`
|
||||
// Direction: claimed_fast_now_miss = stored fast, runtime residual;
|
||||
// mirror_missed = stored residual, runtime fast.
|
||||
Direction string `json:"direction"`
|
||||
Grammar string `json:"grammar,omitempty"`
|
||||
// ShapeDeclined is true when at least one stage-0 grammar matched the
|
||||
// utterance's shape but refused the content (falls through like the router).
|
||||
ShapeDeclined bool `json:"shape_declined"`
|
||||
// DeclinedGrammars names every stage-0 grammar that matched the shape but
|
||||
// declined the content, for claimed_fast_now_miss rows.
|
||||
DeclinedGrammars []string `json:"declined_grammars,omitempty"`
|
||||
}
|
||||
|
||||
type report struct {
|
||||
Meta metaSummary `json:"meta"`
|
||||
Pop popSummary `json:"population"`
|
||||
Conflicts []conflict `json:"conflicts"`
|
||||
ByRoute map[string]map[string]int `json:"by_route"`
|
||||
ByGrammar map[string]int `json:"by_grammar"`
|
||||
BySource map[string]map[string]int `json:"by_source"`
|
||||
ByFamily map[string]map[string]int `json:"by_family"`
|
||||
Direction map[string]int `json:"by_direction"`
|
||||
Declined int `json:"claimed_fast_with_declined_shape"`
|
||||
NoShape int `json:"claimed_fast_with_no_shape"`
|
||||
}
|
||||
|
||||
type metaSummary struct {
|
||||
Total int `json:"total"`
|
||||
DevCount int `json:"dev_count"`
|
||||
Frozen int `json:"frozen_count"`
|
||||
}
|
||||
|
||||
type popSummary struct {
|
||||
StoredFast int `json:"stored_fast"`
|
||||
StoredResid int `json:"stored_residual"`
|
||||
DerivedFast int `json:"derived_fast"`
|
||||
DerivedResid int `json:"derived_residual"`
|
||||
// Dev-pool residual route counts derived as the router sees them today.
|
||||
DevResidualByRoute map[string]int `json:"dev_residual_by_route"`
|
||||
// Dev-pool residual non-action + action OOD as derived.
|
||||
DevResidualNonAction int `json:"dev_residual_non_action"`
|
||||
DevResidualAction int `json:"dev_residual_action"`
|
||||
// Fast rows in the dev pool, derived.
|
||||
DevFast int `json:"dev_fast"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
outPath := flag.String("out", "/tmp/mvn-s23/drift.json", "output JSON path")
|
||||
flag.Parse()
|
||||
|
||||
exs, err := semantic.LoadCorpus()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "load corpus: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
_, dev, _ := semantic.FrozenHoldoutSplit(exs)
|
||||
devSet := make(map[string]bool, len(dev))
|
||||
for _, e := range dev {
|
||||
devSet[e.SourceID] = true
|
||||
}
|
||||
|
||||
// Stage-zero grammar list for shape/declined attribution (same list the
|
||||
// derivation walks).
|
||||
acts := router.DefaultActMatcher{Fns: semantic.ExperimentActVerbs()}
|
||||
gs := router.StageZeroGrammars(acts)
|
||||
|
||||
var (
|
||||
conflicts []conflict
|
||||
byRoute = map[string]map[string]int{}
|
||||
byGrammar = map[string]int{}
|
||||
bySource = map[string]map[string]int{}
|
||||
byFamily = map[string]map[string]int{}
|
||||
byDirection = map[string]int{}
|
||||
storedFast, derivedFast, declined, noShape int
|
||||
devResidByRoute = map[string]int{}
|
||||
devResidNonAct, devResidAct, devFast int
|
||||
)
|
||||
|
||||
for _, e := range exs {
|
||||
o := semantic.DeriveFastPath(e.Text)
|
||||
inDev := devSet[e.SourceID]
|
||||
|
||||
st := e.FastPathResolved
|
||||
if st {
|
||||
storedFast++
|
||||
}
|
||||
if o.Matched {
|
||||
derivedFast++
|
||||
}
|
||||
|
||||
var c *conflict
|
||||
switch {
|
||||
case st && o.Matched:
|
||||
case st && !o.Matched:
|
||||
// Stored fast but the runtime misses. Attribute why.
|
||||
shapeDeclined := false
|
||||
var declinedNames []string
|
||||
for _, g := range gs {
|
||||
_, matched, ok := g.Evaluate(e.Text)
|
||||
if matched && !ok {
|
||||
shapeDeclined = true
|
||||
declinedNames = append(declinedNames, g.Name)
|
||||
}
|
||||
}
|
||||
if shapeDeclined {
|
||||
declined++
|
||||
} else {
|
||||
noShape++
|
||||
}
|
||||
c = &conflict{Direction: "claimed_fast_now_miss", ShapeDeclined: shapeDeclined, DeclinedGrammars: declinedNames}
|
||||
case !st && o.Matched:
|
||||
c = &conflict{Direction: "mirror_missed", Grammar: o.Grammar}
|
||||
}
|
||||
|
||||
if c != nil {
|
||||
c.Text = e.Text
|
||||
c.SourceID = e.SourceID
|
||||
c.Route = string(e.Route)
|
||||
c.Source = e.Source
|
||||
c.Group = e.SplitGroup
|
||||
c.Tags = e.Tags
|
||||
c.Dev = inDev
|
||||
conflicts = append(conflicts, *c)
|
||||
byDirection[c.Direction]++
|
||||
byGrammar[c.Grammar]++
|
||||
if byRoute[c.Direction] == nil {
|
||||
byRoute[c.Direction] = map[string]int{}
|
||||
}
|
||||
byRoute[c.Direction][c.Route]++
|
||||
if bySource[c.Direction] == nil {
|
||||
bySource[c.Direction] = map[string]int{}
|
||||
}
|
||||
bySource[c.Direction][c.Source]++
|
||||
if byFamily[c.Direction] == nil {
|
||||
byFamily[c.Direction] = map[string]int{}
|
||||
}
|
||||
byFamily[c.Direction][c.Group]++
|
||||
}
|
||||
|
||||
if inDev {
|
||||
if o.Matched {
|
||||
devFast++
|
||||
} else {
|
||||
devResidByRoute[string(e.Route)]++
|
||||
if e.Route == semantic.RouteAction {
|
||||
devResidAct++
|
||||
} else {
|
||||
devResidNonAct++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(conflicts, func(i, j int) bool { return conflicts[i].SourceID < conflicts[j].SourceID })
|
||||
|
||||
rep := report{
|
||||
Meta: metaSummary{Total: len(exs), DevCount: len(dev), Frozen: len(exs) - len(dev)},
|
||||
Pop: popSummary{
|
||||
StoredFast: storedFast, StoredResid: len(exs) - storedFast,
|
||||
DerivedFast: derivedFast, DerivedResid: len(exs) - derivedFast,
|
||||
DevResidualByRoute: devResidByRoute,
|
||||
DevResidualNonAction: devResidNonAct, DevResidualAction: devResidAct,
|
||||
DevFast: devFast,
|
||||
},
|
||||
Conflicts: conflicts,
|
||||
ByRoute: byRoute, ByGrammar: byGrammar, BySource: bySource, ByFamily: byFamily,
|
||||
Direction: byDirection, Declined: declined, NoShape: noShape,
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(rep, "", " ")
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "marshal: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := os.WriteFile(*outPath, data, 0644); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "write %s: %v\n", *outPath, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fmt.Printf("total %d (dev %d, frozen %d)\n", rep.Meta.Total, rep.Meta.DevCount, rep.Meta.Frozen)
|
||||
fmt.Printf("stored fast=%d residual=%d\n", rep.Pop.StoredFast, rep.Pop.StoredResid)
|
||||
fmt.Printf("derived fast=%d residual=%d\n", rep.Pop.DerivedFast, rep.Pop.DerivedResid)
|
||||
fmt.Printf("disagreements total %d\n", len(conflicts))
|
||||
for _, d := range []string{"claimed_fast_now_miss", "mirror_missed"} {
|
||||
fmt.Printf(" %-22s %d\n", d, byDirection[d])
|
||||
if d == "claimed_fast_now_miss" {
|
||||
fmt.Printf(" with declined shape: %d no shape: %d\n", declined, noShape)
|
||||
}
|
||||
}
|
||||
fmt.Println(" mirror-missed by grammar:")
|
||||
for _, k := range sortedKeys(byGrammar) {
|
||||
fmt.Printf(" %-28s %d\n", k, byGrammar[k])
|
||||
}
|
||||
fmt.Println(" by route:")
|
||||
for _, d := range sortedKeys(byRoute) {
|
||||
fmt.Printf(" %-22s %v\n", d, byRoute[d])
|
||||
}
|
||||
fmt.Printf("dev pool derived: fast=%d residual=%d (non-action=%d action=%d)\n",
|
||||
rep.Pop.DevFast, rep.Pop.DevResidualNonAction+rep.Pop.DevResidualAction,
|
||||
rep.Pop.DevResidualNonAction, rep.Pop.DevResidualAction)
|
||||
fmt.Printf("dev residual by route: %v\n", rep.Pop.DevResidualByRoute)
|
||||
fmt.Printf("wrote %s\n", *outPath)
|
||||
}
|
||||
|
||||
func sortedKeys[T any](m map[string]T) []string {
|
||||
ks := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
ks = append(ks, k)
|
||||
}
|
||||
sort.Strings(ks)
|
||||
return ks
|
||||
}
|
||||
Reference in New Issue
Block a user