243 lines
7.4 KiB
Go
243 lines
7.4 KiB
Go
// slice23 — fast-path metadata reconciliation diagnostic.
|
|
//
|
|
// Classifies every disagreement between the corpus's stored fast_path_resolved
|
|
// flag and what the production fast path derives today (TryFastPath over the
|
|
// stage-0 grammars with the experiment's act allowlist). Outputs a JSON
|
|
// decomposition and a console summary for docs/evals reports.
|
|
//
|
|
// Usage:
|
|
//
|
|
// go run ./cmd/semantic-router-experiment/slice23/ -out /tmp/mvn-s23/drift.json
|
|
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"os"
|
|
"sort"
|
|
|
|
"github.com/kami/maven/internal/router"
|
|
"github.com/kami/maven/internal/router/semantic"
|
|
)
|
|
|
|
type conflict struct {
|
|
Text string `json:"text"`
|
|
SourceID string `json:"source_id"`
|
|
Route string `json:"route"`
|
|
Source string `json:"source"`
|
|
Group string `json:"split_group"`
|
|
Tags []string `json:"tags,omitempty"`
|
|
Dev bool `json:"dev"`
|
|
// Direction: claimed_fast_now_miss = stored fast, runtime residual;
|
|
// mirror_missed = stored residual, runtime fast.
|
|
Direction string `json:"direction"`
|
|
Grammar string `json:"grammar,omitempty"`
|
|
// ShapeDeclined is true when at least one stage-0 grammar matched the
|
|
// utterance's shape but refused the content (falls through like the router).
|
|
ShapeDeclined bool `json:"shape_declined"`
|
|
// DeclinedGrammars names every stage-0 grammar that matched the shape but
|
|
// declined the content, for claimed_fast_now_miss rows.
|
|
DeclinedGrammars []string `json:"declined_grammars,omitempty"`
|
|
}
|
|
|
|
type report struct {
|
|
Meta metaSummary `json:"meta"`
|
|
Pop popSummary `json:"population"`
|
|
Conflicts []conflict `json:"conflicts"`
|
|
ByRoute map[string]map[string]int `json:"by_route"`
|
|
ByGrammar map[string]int `json:"by_grammar"`
|
|
BySource map[string]map[string]int `json:"by_source"`
|
|
ByFamily map[string]map[string]int `json:"by_family"`
|
|
Direction map[string]int `json:"by_direction"`
|
|
Declined int `json:"claimed_fast_with_declined_shape"`
|
|
NoShape int `json:"claimed_fast_with_no_shape"`
|
|
}
|
|
|
|
type metaSummary struct {
|
|
Total int `json:"total"`
|
|
DevCount int `json:"dev_count"`
|
|
Frozen int `json:"frozen_count"`
|
|
}
|
|
|
|
type popSummary struct {
|
|
StoredFast int `json:"stored_fast"`
|
|
StoredResid int `json:"stored_residual"`
|
|
DerivedFast int `json:"derived_fast"`
|
|
DerivedResid int `json:"derived_residual"`
|
|
// Dev-pool residual route counts derived as the router sees them today.
|
|
DevResidualByRoute map[string]int `json:"dev_residual_by_route"`
|
|
// Dev-pool residual non-action + action OOD as derived.
|
|
DevResidualNonAction int `json:"dev_residual_non_action"`
|
|
DevResidualAction int `json:"dev_residual_action"`
|
|
// Fast rows in the dev pool, derived.
|
|
DevFast int `json:"dev_fast"`
|
|
}
|
|
|
|
func main() {
|
|
outPath := flag.String("out", "/tmp/mvn-s23/drift.json", "output JSON path")
|
|
flag.Parse()
|
|
|
|
exs, err := semantic.LoadCorpus()
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "load corpus: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
|
|
_, dev, _ := semantic.FrozenHoldoutSplit(exs)
|
|
devSet := make(map[string]bool, len(dev))
|
|
for _, e := range dev {
|
|
devSet[e.SourceID] = true
|
|
}
|
|
|
|
// Stage-zero grammar list for shape/declined attribution (same list the
|
|
// derivation walks).
|
|
acts := router.DefaultActMatcher{Fns: semantic.ExperimentActVerbs()}
|
|
gs := router.StageZeroGrammars(acts)
|
|
|
|
var (
|
|
conflicts []conflict
|
|
byRoute = map[string]map[string]int{}
|
|
byGrammar = map[string]int{}
|
|
bySource = map[string]map[string]int{}
|
|
byFamily = map[string]map[string]int{}
|
|
byDirection = map[string]int{}
|
|
storedFast, derivedFast, declined, noShape int
|
|
devResidByRoute = map[string]int{}
|
|
devResidNonAct, devResidAct, devFast int
|
|
)
|
|
|
|
for _, e := range exs {
|
|
o := semantic.DeriveFastPath(e.Text)
|
|
inDev := devSet[e.SourceID]
|
|
|
|
st := e.FastPathResolved
|
|
if st {
|
|
storedFast++
|
|
}
|
|
if o.Matched {
|
|
derivedFast++
|
|
}
|
|
|
|
var c *conflict
|
|
switch {
|
|
case st && o.Matched:
|
|
case st && !o.Matched:
|
|
// Stored fast but the runtime misses. Attribute why.
|
|
shapeDeclined := false
|
|
var declinedNames []string
|
|
for _, g := range gs {
|
|
_, matched, ok := g.Evaluate(e.Text)
|
|
if matched && !ok {
|
|
shapeDeclined = true
|
|
declinedNames = append(declinedNames, g.Name)
|
|
}
|
|
}
|
|
if shapeDeclined {
|
|
declined++
|
|
} else {
|
|
noShape++
|
|
}
|
|
c = &conflict{Direction: "claimed_fast_now_miss", ShapeDeclined: shapeDeclined, DeclinedGrammars: declinedNames}
|
|
case !st && o.Matched:
|
|
c = &conflict{Direction: "mirror_missed", Grammar: o.Grammar}
|
|
}
|
|
|
|
if c != nil {
|
|
c.Text = e.Text
|
|
c.SourceID = e.SourceID
|
|
c.Route = string(e.Route)
|
|
c.Source = e.Source
|
|
c.Group = e.SplitGroup
|
|
c.Tags = e.Tags
|
|
c.Dev = inDev
|
|
conflicts = append(conflicts, *c)
|
|
byDirection[c.Direction]++
|
|
byGrammar[c.Grammar]++
|
|
if byRoute[c.Direction] == nil {
|
|
byRoute[c.Direction] = map[string]int{}
|
|
}
|
|
byRoute[c.Direction][c.Route]++
|
|
if bySource[c.Direction] == nil {
|
|
bySource[c.Direction] = map[string]int{}
|
|
}
|
|
bySource[c.Direction][c.Source]++
|
|
if byFamily[c.Direction] == nil {
|
|
byFamily[c.Direction] = map[string]int{}
|
|
}
|
|
byFamily[c.Direction][c.Group]++
|
|
}
|
|
|
|
if inDev {
|
|
if o.Matched {
|
|
devFast++
|
|
} else {
|
|
devResidByRoute[string(e.Route)]++
|
|
if e.Route == semantic.RouteAction {
|
|
devResidAct++
|
|
} else {
|
|
devResidNonAct++
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
sort.Slice(conflicts, func(i, j int) bool { return conflicts[i].SourceID < conflicts[j].SourceID })
|
|
|
|
rep := report{
|
|
Meta: metaSummary{Total: len(exs), DevCount: len(dev), Frozen: len(exs) - len(dev)},
|
|
Pop: popSummary{
|
|
StoredFast: storedFast, StoredResid: len(exs) - storedFast,
|
|
DerivedFast: derivedFast, DerivedResid: len(exs) - derivedFast,
|
|
DevResidualByRoute: devResidByRoute,
|
|
DevResidualNonAction: devResidNonAct, DevResidualAction: devResidAct,
|
|
DevFast: devFast,
|
|
},
|
|
Conflicts: conflicts,
|
|
ByRoute: byRoute, ByGrammar: byGrammar, BySource: bySource, ByFamily: byFamily,
|
|
Direction: byDirection, Declined: declined, NoShape: noShape,
|
|
}
|
|
|
|
data, err := json.MarshalIndent(rep, "", " ")
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "marshal: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
if err := os.WriteFile(*outPath, data, 0644); err != nil {
|
|
fmt.Fprintf(os.Stderr, "write %s: %v\n", *outPath, err)
|
|
os.Exit(1)
|
|
}
|
|
|
|
fmt.Printf("total %d (dev %d, frozen %d)\n", rep.Meta.Total, rep.Meta.DevCount, rep.Meta.Frozen)
|
|
fmt.Printf("stored fast=%d residual=%d\n", rep.Pop.StoredFast, rep.Pop.StoredResid)
|
|
fmt.Printf("derived fast=%d residual=%d\n", rep.Pop.DerivedFast, rep.Pop.DerivedResid)
|
|
fmt.Printf("disagreements total %d\n", len(conflicts))
|
|
for _, d := range []string{"claimed_fast_now_miss", "mirror_missed"} {
|
|
fmt.Printf(" %-22s %d\n", d, byDirection[d])
|
|
if d == "claimed_fast_now_miss" {
|
|
fmt.Printf(" with declined shape: %d no shape: %d\n", declined, noShape)
|
|
}
|
|
}
|
|
fmt.Println(" mirror-missed by grammar:")
|
|
for _, k := range sortedKeys(byGrammar) {
|
|
fmt.Printf(" %-28s %d\n", k, byGrammar[k])
|
|
}
|
|
fmt.Println(" by route:")
|
|
for _, d := range sortedKeys(byRoute) {
|
|
fmt.Printf(" %-22s %v\n", d, byRoute[d])
|
|
}
|
|
fmt.Printf("dev pool derived: fast=%d residual=%d (non-action=%d action=%d)\n",
|
|
rep.Pop.DevFast, rep.Pop.DevResidualNonAction+rep.Pop.DevResidualAction,
|
|
rep.Pop.DevResidualNonAction, rep.Pop.DevResidualAction)
|
|
fmt.Printf("dev residual by route: %v\n", rep.Pop.DevResidualByRoute)
|
|
fmt.Printf("wrote %s\n", *outPath)
|
|
}
|
|
|
|
func sortedKeys[T any](m map[string]T) []string {
|
|
ks := make([]string, 0, len(m))
|
|
for k := range m {
|
|
ks = append(ks, k)
|
|
}
|
|
sort.Strings(ks)
|
|
return ks
|
|
} |