router/semantic: slice 23 corpus fast-path reconciliation — DeriveFastPath over the real router replaces the regex mirror, factory/merge validated, corpus rebuilt (dataset_hash unchanged), drift diagnostic and rerun tooling
This commit is contained in:
@@ -15,7 +15,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"regexp"
|
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@@ -1089,85 +1088,11 @@ func generatorQuestion(seed semantic.SemanticSeed) []semantic.GeneratedSurface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ─── Fast-path classifier ────────────────────────────────────────────────────
|
// ─── Fast-path classifier ────────────────────────────────────────────────────
|
||||||
// Simplified pattern matcher that mirrors stage-0 grammar outcomes.
|
// Fast-path metadata is derived from the real router, not a regex mirror:
|
||||||
|
// DeriveFastPath runs the production TryFastPath over the stage-0 grammars
|
||||||
var (
|
// with the experiment's act allowlist (see internal/router/semantic/fastpath.go).
|
||||||
reminderVerbRe = regexp.MustCompile(`(?i)^\s*(напомни|напомните|напомнить|напоминай|разбуди|разбудите|разбудить|буди|remind|wake)\b`)
|
// A hand-maintained pattern list drifted from the grammars and labelled 185
|
||||||
timeQueryRe = regexp.MustCompile(`(?i)^\s*(сколько\s+сейчас\s+времени|который\s+час|какое\s+число|какой\s+день|what\s+time|what's\s+the\s+date)\b`)
|
// residual rows as fast (docs/evals/2026-09-08-slice22-*); it is removed.
|
||||||
captureVerbRe = regexp.MustCompile(`(?i)^\s*(запиши|запомни|отметь|заметь|добавь|сохрани|занеси|внеси|note|remember|log|save|add)\b`)
|
|
||||||
praxisItemRe = regexp.MustCompile(`(?i)\b(item[_-][a-z0-9_-]+)`)
|
|
||||||
quietOnRe = regexp.MustCompile(`(?i)(тихий\s+режим|режим\s+тишина|тише|потише|quiet\s+(mode|on))`)
|
|
||||||
quietOffRe = regexp.MustCompile(`(?i)(хватит\s+тихого|громкий\s+режим|выключи\s+тихий|quiet\s+(off|end))`)
|
|
||||||
taskDoneRe = regexp.MustCompile(`(?i)^\s*(закрой\s+задачу|заверши\s+задачу|close\s+the\s+task|finish\s+the\s+task)\b`)
|
|
||||||
taskDropRe = regexp.MustCompile(`(?i)^\s*(убери\s+задачу|удали\s+задачу|отмени\s+задачу|drop\s+the\s+task|remove\s+the\s+task)\b`)
|
|
||||||
)
|
|
||||||
|
|
||||||
// tool aliases → fast-path matchers (subset of what DefaultActMatcher would match)
|
|
||||||
var toolAliasRe = []*regexp.Regexp{
|
|
||||||
regexp.MustCompile(`(?i)^\s*(перезапусти|перезагрузи|рестарт)\b`),
|
|
||||||
regexp.MustCompile(`(?i)^\s*(останови|выключи)\b`),
|
|
||||||
regexp.MustCompile(`(?i)^\s*(запусти|старт)\b`),
|
|
||||||
regexp.MustCompile(`(?i)^\s*(перезапусти\s+контейнер|перезагрузи\s+контейнер)\b`),
|
|
||||||
regexp.MustCompile(`(?i)^\s*(останови\s+контейнер)\b`),
|
|
||||||
regexp.MustCompile(`(?i)^\s*(перезагрузи\s+сервер|перезагрузи\s+хост|ребут)\b`),
|
|
||||||
regexp.MustCompile(`(?i)^\s*(покажи\s+статус|проверь\s+статус|статус)\b`),
|
|
||||||
regexp.MustCompile(`(?i)^\s*(покажи\s+контейнеры|список\s+контейнеров|что\s+запущено)\b`),
|
|
||||||
regexp.MustCompile(`(?i)^\s*(покажи\s+uptime|аптайм|как\s+работает\s+сервер)\b`),
|
|
||||||
regexp.MustCompile(`(?i)^\s*(покажи\s+диск|сколько\s+места\s+на\s+диске)\b`),
|
|
||||||
regexp.MustCompile(`(?i)^\s*(покажи\s+память|свободная\s+память)\b`),
|
|
||||||
regexp.MustCompile(`(?i)^\s*(покажи\s+логи|логи|лог)\b`),
|
|
||||||
}
|
|
||||||
|
|
||||||
// praxis lifecycle verbs (bare, without item reference → residual action)
|
|
||||||
var praxisLifecycleRe = regexp.MustCompile(`(?i)^\s*(закрой|закрыть|resolve|close|принято|принять|acknowledge|ack|игнорируй|игнорировать|пропусти|ignore|skip|закрепи|закрепить|pin)\b`)
|
|
||||||
var praxisAttentionRe = regexp.MustCompile(`(?i)^\s*(что\s+требует\s+внимания|что\s+нового\s+по\s+(задачам|проектам|сервисам)|needs\s+attention)\b`)
|
|
||||||
var praxisChangesRe = regexp.MustCompile(`(?i)^\s*(что\s+изменилось|какие\s+изменения|changed|changes)\b`)
|
|
||||||
var praxisEntityRe = regexp.MustCompile(`(?i)^\s*(статус|status|как\s+дела\s+у|how\s+is)\s+`)
|
|
||||||
|
|
||||||
// classifyFastPath determines whether a surface would be handled by stage-0.
|
|
||||||
func classifyFastPath(text string) bool {
|
|
||||||
t := strings.TrimSpace(text)
|
|
||||||
|
|
||||||
// Reminder verbs → always fast-path
|
|
||||||
if reminderVerbRe.MatchString(t) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
// Time/date queries → fast-path
|
|
||||||
if timeQueryRe.MatchString(t) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
// Task-status commands → fast-path
|
|
||||||
if taskDoneRe.MatchString(t) || taskDropRe.MatchString(t) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
// Tool aliases → fast-path (only for verbs alone, not when combined with objects in ways the matcher wouldn't handle)
|
|
||||||
for _, re := range toolAliasRe {
|
|
||||||
if re.MatchString(t) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Praxis lifecycle with item reference → fast-path
|
|
||||||
if praxisLifecycleRe.MatchString(t) && praxisItemRe.MatchString(t) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
// Praxis attention/changes → fast-path
|
|
||||||
if praxisAttentionRe.MatchString(t) || praxisChangesRe.MatchString(t) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
// Praxis entity attention → fast-path
|
|
||||||
if praxisEntityRe.MatchString(t) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
// Quiet mode → fast-path (resolved pre-route)
|
|
||||||
if quietOnRe.MatchString(t) || quietOffRe.MatchString(t) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
// Capture verbs → some are fast-path (note capture)
|
|
||||||
if captureVerbRe.MatchString(t) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Helpers ─────────────────────────────────────────────────────────────────
|
// ─── Helpers ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -1241,7 +1166,7 @@ func buildCorpus(seeds []semantic.SemanticSeed) []semantic.RouteExample {
|
|||||||
exampleCounter++
|
exampleCounter++
|
||||||
srcID := fmt.Sprintf("%s-%03d", seed.ID, exampleCounter)
|
srcID := fmt.Sprintf("%s-%03d", seed.ID, exampleCounter)
|
||||||
|
|
||||||
fp := classifyFastPath(surf.Text)
|
fp := semantic.DeriveFastPath(surf.Text).Matched
|
||||||
isResidual := !fp
|
isResidual := !fp
|
||||||
|
|
||||||
tags := append([]string{}, seed.Tags...)
|
tags := append([]string{}, seed.Tags...)
|
||||||
|
|||||||
@@ -92,13 +92,37 @@ func main() {
|
|||||||
}
|
}
|
||||||
fmt.Fprintf(os.Stderr, "merged: %d examples (skipped %d duplicates)\n", len(merged), skipped)
|
fmt.Fprintf(os.Stderr, "merged: %d examples (skipped %d duplicates)\n", len(merged), skipped)
|
||||||
|
|
||||||
// 5. Validate
|
// 5. Fast-path metadata must match the real router. V2 rows are derived
|
||||||
|
// by interactively-labeled construction; a stale manually-supplied value
|
||||||
|
// is a build error, never silently rewritten. Frozen holdout rows are
|
||||||
|
// preserved verbatim — report drift, do not touch them.
|
||||||
|
v2Checked, frozenChecked, frozenDrift := 0, 0, 0
|
||||||
|
for _, e := range merged {
|
||||||
|
derived := semantic.DeriveFastPath(e.Text).Matched
|
||||||
|
if frozenTexts[strings.TrimSpace(e.Text)] {
|
||||||
|
frozenChecked++
|
||||||
|
if derived != e.FastPathResolved {
|
||||||
|
frozenDrift++
|
||||||
|
fmt.Fprintf(os.Stderr, "frozen drift: %q stored=%v derived=%v\n", e.Text, e.FastPathResolved, derived)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
v2Checked++
|
||||||
|
if derived != e.FastPathResolved {
|
||||||
|
log.Fatalf("v2 row (source=%s source_id=%s) fast_path_resolved=%v but router derives %v: %q",
|
||||||
|
e.Source, e.SourceID, e.FastPathResolved, derived, e.Text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Fprintf(os.Stderr, "fast-path check: v2 rows %d OK, frozen rows %d (drift %d, reported only)\n",
|
||||||
|
v2Checked, frozenChecked, frozenDrift)
|
||||||
|
|
||||||
|
// 6. Validate
|
||||||
if err := semantic.ValidateCorpus(merged); err != nil {
|
if err := semantic.ValidateCorpus(merged); err != nil {
|
||||||
log.Fatalf("validation failed: %v", err)
|
log.Fatalf("validation failed: %v", err)
|
||||||
}
|
}
|
||||||
fmt.Fprintf(os.Stderr, "validation: OK\n")
|
fmt.Fprintf(os.Stderr, "validation: OK\n")
|
||||||
|
|
||||||
// 6. Compute dataset hash
|
// 7. Compute dataset hash
|
||||||
texts := make([]string, len(merged))
|
texts := make([]string, len(merged))
|
||||||
for i, e := range merged {
|
for i, e := range merged {
|
||||||
texts[i] = e.Text
|
texts[i] = e.Text
|
||||||
@@ -107,7 +131,7 @@ func main() {
|
|||||||
h := sha256.Sum256([]byte(strings.Join(texts, "\n")))
|
h := sha256.Sum256([]byte(strings.Join(texts, "\n")))
|
||||||
datasetHash := hex.EncodeToString(h[:16])
|
datasetHash := hex.EncodeToString(h[:16])
|
||||||
|
|
||||||
// 7. Stats
|
// 8. Stats
|
||||||
routeCounts := make(map[semantic.SemanticRoute]int)
|
routeCounts := make(map[semantic.SemanticRoute]int)
|
||||||
fpCount, resCount := 0, 0
|
fpCount, resCount := 0, 0
|
||||||
for _, e := range merged {
|
for _, e := range merged {
|
||||||
@@ -124,7 +148,7 @@ func main() {
|
|||||||
}
|
}
|
||||||
fmt.Fprintf(os.Stderr, "fast-path: %d residual: %d\n", fpCount, resCount)
|
fmt.Fprintf(os.Stderr, "fast-path: %d residual: %d\n", fpCount, resCount)
|
||||||
|
|
||||||
// 8. Write merged corpus
|
// 9. Write merged corpus
|
||||||
outEnv := semantic.CorpusEnvelope{
|
outEnv := semantic.CorpusEnvelope{
|
||||||
SchemaVersion: 1,
|
SchemaVersion: 1,
|
||||||
Name: "semantic_coarse_route_v1",
|
Name: "semantic_coarse_route_v1",
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
|
|
||||||
"github.com/kami/maven/internal/router"
|
"github.com/kami/maven/internal/router"
|
||||||
|
"github.com/kami/maven/internal/router/semantic"
|
||||||
)
|
)
|
||||||
|
|
||||||
// headsMain runs the deployed cascade minus the resident LLM: stage-0
|
// headsMain runs the deployed cascade minus the resident LLM: stage-0
|
||||||
@@ -48,7 +49,7 @@ func headsMain(poolPath, outPath string) {
|
|||||||
}
|
}
|
||||||
defer heads.Close()
|
defer heads.Close()
|
||||||
|
|
||||||
acts := router.DefaultActMatcher{Fns: actVerbList()}
|
acts := router.DefaultActMatcher{Fns: semantic.ExperimentActVerbs()}
|
||||||
r := router.New(router.Config{
|
r := router.New(router.Config{
|
||||||
Grammars: router.StageZeroGrammars(acts),
|
Grammars: router.StageZeroGrammars(acts),
|
||||||
Classifier: cls,
|
Classifier: cls,
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/kami/maven/internal/router"
|
"github.com/kami/maven/internal/router"
|
||||||
|
"github.com/kami/maven/internal/router/semantic"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Seed loading replicated from cmd/mavend/voicewire.go (seedClassifier) and
|
// Seed loading replicated from cmd/mavend/voicewire.go (seedClassifier) and
|
||||||
@@ -23,26 +24,13 @@ var seedIntents = []router.Intent{
|
|||||||
router.IntentNote, router.IntentQuery, router.IntentChat, router.IntentSystem,
|
router.IntentNote, router.IntentQuery, router.IntentChat, router.IntentSystem,
|
||||||
}
|
}
|
||||||
|
|
||||||
// actVerbList mirrors the eval fixture's static allowlist
|
|
||||||
// (internal/router/semantic/helpers_test.go). The production matcher's
|
|
||||||
// allowlist is the deployment's enabled tools; an act grammar can only catch
|
|
||||||
// a row whose first tokens match an allowlisted verb, and the experiment's
|
|
||||||
// act allowlist is the one the accepted slice-21 methodology used.
|
|
||||||
func actVerbList() []string {
|
|
||||||
return []string{
|
|
||||||
"перезапусти", "перезагрузи", "выключи", "включи", "останови", "запусти",
|
|
||||||
"закрой", "открой", "сделай", "поставь",
|
|
||||||
"restart", "reboot", "stop", "start", "turn off", "turn on", "open", "close",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildMinimalRouter reproduces internal/router/semantic/buildMinimalRouter:
|
// buildMinimalRouter reproduces internal/router/semantic/buildMinimalRouter:
|
||||||
// the daemon's grammar set, a hash-embedder classifier seeded from
|
// the daemon's grammar set, a hash-embedder classifier seeded from
|
||||||
// models/seeds, and the deployed 0.55 threshold. Deterministic and
|
// models/seeds, and the deployed 0.55 threshold. Deterministic and
|
||||||
// reproducible. The ONNX embedder and the routing heads score elsewhere;
|
// reproducible. The ONNX embedder and the routing heads score elsewhere;
|
||||||
// this is the floor the eval fixture reports as the legacy baseline.
|
// this is the floor the eval fixture reports as the legacy baseline.
|
||||||
func buildMinimalRouter() *router.Router {
|
func buildMinimalRouter() *router.Router {
|
||||||
acts := router.DefaultActMatcher{Fns: actVerbList()}
|
acts := router.DefaultActMatcher{Fns: semantic.ExperimentActVerbs()}
|
||||||
cls := router.NewClassifier(router.NewHashEmbedder(1024))
|
cls := router.NewClassifier(router.NewHashEmbedder(1024))
|
||||||
seedClassifier(cls)
|
seedClassifier(cls)
|
||||||
return router.New(router.Config{
|
return router.New(router.Config{
|
||||||
|
|||||||
@@ -0,0 +1,243 @@
|
|||||||
|
// slice23 — fast-path metadata reconciliation diagnostic.
|
||||||
|
//
|
||||||
|
// Classifies every disagreement between the corpus's stored fast_path_resolved
|
||||||
|
// flag and what the production fast path derives today (TryFastPath over the
|
||||||
|
// stage-0 grammars with the experiment's act allowlist). Outputs a JSON
|
||||||
|
// decomposition and a console summary for docs/evals reports.
|
||||||
|
//
|
||||||
|
// Usage:
|
||||||
|
//
|
||||||
|
// go run ./cmd/semantic-router-experiment/slice23/ -out /tmp/mvn-s23/drift.json
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"github.com/kami/maven/internal/router"
|
||||||
|
"github.com/kami/maven/internal/router/semantic"
|
||||||
|
)
|
||||||
|
|
||||||
|
type conflict struct {
|
||||||
|
Text string `json:"text"`
|
||||||
|
SourceID string `json:"source_id"`
|
||||||
|
Route string `json:"route"`
|
||||||
|
Source string `json:"source"`
|
||||||
|
Group string `json:"split_group"`
|
||||||
|
Tags []string `json:"tags,omitempty"`
|
||||||
|
Dev bool `json:"dev"`
|
||||||
|
// Direction: claimed_fast_now_miss = stored fast, runtime residual;
|
||||||
|
// mirror_missed = stored residual, runtime fast.
|
||||||
|
Direction string `json:"direction"`
|
||||||
|
Grammar string `json:"grammar,omitempty"`
|
||||||
|
// ShapeDeclined is true when at least one stage-0 grammar matched the
|
||||||
|
// utterance's shape but refused the content (falls through like the router).
|
||||||
|
ShapeDeclined bool `json:"shape_declined"`
|
||||||
|
// DeclinedGrammars names every stage-0 grammar that matched the shape but
|
||||||
|
// declined the content, for claimed_fast_now_miss rows.
|
||||||
|
DeclinedGrammars []string `json:"declined_grammars,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type report struct {
|
||||||
|
Meta metaSummary `json:"meta"`
|
||||||
|
Pop popSummary `json:"population"`
|
||||||
|
Conflicts []conflict `json:"conflicts"`
|
||||||
|
ByRoute map[string]map[string]int `json:"by_route"`
|
||||||
|
ByGrammar map[string]int `json:"by_grammar"`
|
||||||
|
BySource map[string]map[string]int `json:"by_source"`
|
||||||
|
ByFamily map[string]map[string]int `json:"by_family"`
|
||||||
|
Direction map[string]int `json:"by_direction"`
|
||||||
|
Declined int `json:"claimed_fast_with_declined_shape"`
|
||||||
|
NoShape int `json:"claimed_fast_with_no_shape"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type metaSummary struct {
|
||||||
|
Total int `json:"total"`
|
||||||
|
DevCount int `json:"dev_count"`
|
||||||
|
Frozen int `json:"frozen_count"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type popSummary struct {
|
||||||
|
StoredFast int `json:"stored_fast"`
|
||||||
|
StoredResid int `json:"stored_residual"`
|
||||||
|
DerivedFast int `json:"derived_fast"`
|
||||||
|
DerivedResid int `json:"derived_residual"`
|
||||||
|
// Dev-pool residual route counts derived as the router sees them today.
|
||||||
|
DevResidualByRoute map[string]int `json:"dev_residual_by_route"`
|
||||||
|
// Dev-pool residual non-action + action OOD as derived.
|
||||||
|
DevResidualNonAction int `json:"dev_residual_non_action"`
|
||||||
|
DevResidualAction int `json:"dev_residual_action"`
|
||||||
|
// Fast rows in the dev pool, derived.
|
||||||
|
DevFast int `json:"dev_fast"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
outPath := flag.String("out", "/tmp/mvn-s23/drift.json", "output JSON path")
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
|
exs, err := semantic.LoadCorpus()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "load corpus: %v\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, dev, _ := semantic.FrozenHoldoutSplit(exs)
|
||||||
|
devSet := make(map[string]bool, len(dev))
|
||||||
|
for _, e := range dev {
|
||||||
|
devSet[e.SourceID] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stage-zero grammar list for shape/declined attribution (same list the
|
||||||
|
// derivation walks).
|
||||||
|
acts := router.DefaultActMatcher{Fns: semantic.ExperimentActVerbs()}
|
||||||
|
gs := router.StageZeroGrammars(acts)
|
||||||
|
|
||||||
|
var (
|
||||||
|
conflicts []conflict
|
||||||
|
byRoute = map[string]map[string]int{}
|
||||||
|
byGrammar = map[string]int{}
|
||||||
|
bySource = map[string]map[string]int{}
|
||||||
|
byFamily = map[string]map[string]int{}
|
||||||
|
byDirection = map[string]int{}
|
||||||
|
storedFast, derivedFast, declined, noShape int
|
||||||
|
devResidByRoute = map[string]int{}
|
||||||
|
devResidNonAct, devResidAct, devFast int
|
||||||
|
)
|
||||||
|
|
||||||
|
for _, e := range exs {
|
||||||
|
o := semantic.DeriveFastPath(e.Text)
|
||||||
|
inDev := devSet[e.SourceID]
|
||||||
|
|
||||||
|
st := e.FastPathResolved
|
||||||
|
if st {
|
||||||
|
storedFast++
|
||||||
|
}
|
||||||
|
if o.Matched {
|
||||||
|
derivedFast++
|
||||||
|
}
|
||||||
|
|
||||||
|
var c *conflict
|
||||||
|
switch {
|
||||||
|
case st && o.Matched:
|
||||||
|
case st && !o.Matched:
|
||||||
|
// Stored fast but the runtime misses. Attribute why.
|
||||||
|
shapeDeclined := false
|
||||||
|
var declinedNames []string
|
||||||
|
for _, g := range gs {
|
||||||
|
_, matched, ok := g.Evaluate(e.Text)
|
||||||
|
if matched && !ok {
|
||||||
|
shapeDeclined = true
|
||||||
|
declinedNames = append(declinedNames, g.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if shapeDeclined {
|
||||||
|
declined++
|
||||||
|
} else {
|
||||||
|
noShape++
|
||||||
|
}
|
||||||
|
c = &conflict{Direction: "claimed_fast_now_miss", ShapeDeclined: shapeDeclined, DeclinedGrammars: declinedNames}
|
||||||
|
case !st && o.Matched:
|
||||||
|
c = &conflict{Direction: "mirror_missed", Grammar: o.Grammar}
|
||||||
|
}
|
||||||
|
|
||||||
|
if c != nil {
|
||||||
|
c.Text = e.Text
|
||||||
|
c.SourceID = e.SourceID
|
||||||
|
c.Route = string(e.Route)
|
||||||
|
c.Source = e.Source
|
||||||
|
c.Group = e.SplitGroup
|
||||||
|
c.Tags = e.Tags
|
||||||
|
c.Dev = inDev
|
||||||
|
conflicts = append(conflicts, *c)
|
||||||
|
byDirection[c.Direction]++
|
||||||
|
byGrammar[c.Grammar]++
|
||||||
|
if byRoute[c.Direction] == nil {
|
||||||
|
byRoute[c.Direction] = map[string]int{}
|
||||||
|
}
|
||||||
|
byRoute[c.Direction][c.Route]++
|
||||||
|
if bySource[c.Direction] == nil {
|
||||||
|
bySource[c.Direction] = map[string]int{}
|
||||||
|
}
|
||||||
|
bySource[c.Direction][c.Source]++
|
||||||
|
if byFamily[c.Direction] == nil {
|
||||||
|
byFamily[c.Direction] = map[string]int{}
|
||||||
|
}
|
||||||
|
byFamily[c.Direction][c.Group]++
|
||||||
|
}
|
||||||
|
|
||||||
|
if inDev {
|
||||||
|
if o.Matched {
|
||||||
|
devFast++
|
||||||
|
} else {
|
||||||
|
devResidByRoute[string(e.Route)]++
|
||||||
|
if e.Route == semantic.RouteAction {
|
||||||
|
devResidAct++
|
||||||
|
} else {
|
||||||
|
devResidNonAct++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Slice(conflicts, func(i, j int) bool { return conflicts[i].SourceID < conflicts[j].SourceID })
|
||||||
|
|
||||||
|
rep := report{
|
||||||
|
Meta: metaSummary{Total: len(exs), DevCount: len(dev), Frozen: len(exs) - len(dev)},
|
||||||
|
Pop: popSummary{
|
||||||
|
StoredFast: storedFast, StoredResid: len(exs) - storedFast,
|
||||||
|
DerivedFast: derivedFast, DerivedResid: len(exs) - derivedFast,
|
||||||
|
DevResidualByRoute: devResidByRoute,
|
||||||
|
DevResidualNonAction: devResidNonAct, DevResidualAction: devResidAct,
|
||||||
|
DevFast: devFast,
|
||||||
|
},
|
||||||
|
Conflicts: conflicts,
|
||||||
|
ByRoute: byRoute, ByGrammar: byGrammar, BySource: bySource, ByFamily: byFamily,
|
||||||
|
Direction: byDirection, Declined: declined, NoShape: noShape,
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := json.MarshalIndent(rep, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "marshal: %v\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(*outPath, data, 0644); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "write %s: %v\n", *outPath, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("total %d (dev %d, frozen %d)\n", rep.Meta.Total, rep.Meta.DevCount, rep.Meta.Frozen)
|
||||||
|
fmt.Printf("stored fast=%d residual=%d\n", rep.Pop.StoredFast, rep.Pop.StoredResid)
|
||||||
|
fmt.Printf("derived fast=%d residual=%d\n", rep.Pop.DerivedFast, rep.Pop.DerivedResid)
|
||||||
|
fmt.Printf("disagreements total %d\n", len(conflicts))
|
||||||
|
for _, d := range []string{"claimed_fast_now_miss", "mirror_missed"} {
|
||||||
|
fmt.Printf(" %-22s %d\n", d, byDirection[d])
|
||||||
|
if d == "claimed_fast_now_miss" {
|
||||||
|
fmt.Printf(" with declined shape: %d no shape: %d\n", declined, noShape)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Println(" mirror-missed by grammar:")
|
||||||
|
for _, k := range sortedKeys(byGrammar) {
|
||||||
|
fmt.Printf(" %-28s %d\n", k, byGrammar[k])
|
||||||
|
}
|
||||||
|
fmt.Println(" by route:")
|
||||||
|
for _, d := range sortedKeys(byRoute) {
|
||||||
|
fmt.Printf(" %-22s %v\n", d, byRoute[d])
|
||||||
|
}
|
||||||
|
fmt.Printf("dev pool derived: fast=%d residual=%d (non-action=%d action=%d)\n",
|
||||||
|
rep.Pop.DevFast, rep.Pop.DevResidualNonAction+rep.Pop.DevResidualAction,
|
||||||
|
rep.Pop.DevResidualNonAction, rep.Pop.DevResidualAction)
|
||||||
|
fmt.Printf("dev residual by route: %v\n", rep.Pop.DevResidualByRoute)
|
||||||
|
fmt.Printf("wrote %s\n", *outPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortedKeys[T any](m map[string]T) []string {
|
||||||
|
ks := make([]string, 0, len(m))
|
||||||
|
for k := range m {
|
||||||
|
ks = append(ks, k)
|
||||||
|
}
|
||||||
|
sort.Strings(ks)
|
||||||
|
return ks
|
||||||
|
}
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Slice 23 emit: five-way residual non-action semantic router — data files
|
||||||
|
========================================================================
|
||||||
|
|
||||||
|
Slice 23 reconciles corpus fast-path metadata with the production router
|
||||||
|
(TryFastPath over stage-0 grammars). The corpus builder no longer mirrors the
|
||||||
|
grammars by hand; fast_path_resolved is derived from the router, so this emit
|
||||||
|
flags exactly the rows the router genuinely leaves for the general cascade.
|
||||||
|
|
||||||
|
This script only repackages the frozen dev pool for the Go legacy baseline and
|
||||||
|
the Python experiment, writing into /tmp/mvn-s23 so the slice-22 artifacts
|
||||||
|
stay untouched. Logic is slice22_emit.py verbatim; only OUT_DIR differs.
|
||||||
|
|
||||||
|
/tmp/mvn-s23/pool.json residual non-action dev rows: idx, text, n_text,
|
||||||
|
route, tags, cv_fold, split_group, family_id,
|
||||||
|
source_id (1509 rows)
|
||||||
|
/tmp/mvn-s23/ood.json residual ACTION dev rows (720): same shape; OOD
|
||||||
|
probes only, never primary metrics
|
||||||
|
/tmp/mvn-s23/stats.json population summary (routes, families, folds)
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
sys.path.insert(0, HERE)
|
||||||
|
|
||||||
|
import slice18_sparse # noqa: E402 (normalize_match_text, load_data, filters)
|
||||||
|
|
||||||
|
OUT_DIR = "/tmp/mvn-s23"
|
||||||
|
|
||||||
|
ROUTES = ["conversation", "knowledge", "memory_write", "system", "uncertain"]
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
meta, examples = slice18_sparse.load_data()
|
||||||
|
dev = slice18_sparse.filter_dev_pool(examples)
|
||||||
|
print(f"dev pool: {len(dev)} rows "
|
||||||
|
f"(meta declares dev_count={meta.get('dev_count')})")
|
||||||
|
|
||||||
|
rows = []
|
||||||
|
for i, e in enumerate(dev):
|
||||||
|
if not e["fast_path_resolved"]:
|
||||||
|
rows.append({
|
||||||
|
"idx": i,
|
||||||
|
"text": e["text"],
|
||||||
|
"n_text": slice18_sparse.normalize_match_text(e["text"]),
|
||||||
|
"route": e["route"],
|
||||||
|
"tags": sorted(set(e.get("tags", []))),
|
||||||
|
"cv_fold": e["cv_fold"],
|
||||||
|
"split_group": e["split_group"],
|
||||||
|
"family_id": e["family_id"],
|
||||||
|
"source_id": e["source_id"],
|
||||||
|
})
|
||||||
|
|
||||||
|
na = [r for r in rows if r["route"] != "action"]
|
||||||
|
ood = [r for r in rows if r["route"] == "action"]
|
||||||
|
print(f"residual rows: {len(rows)} non-action: {len(na)} action(OOD): {len(ood)}")
|
||||||
|
|
||||||
|
by_route = {}
|
||||||
|
for r in na:
|
||||||
|
by_route[r["route"]] = by_route.get(r["route"], 0) + 1
|
||||||
|
print("routes:", by_route)
|
||||||
|
assert sum(by_route.values()) == len(na)
|
||||||
|
assert set(ROUTES) == set(by_route), "route set must be the five-way"
|
||||||
|
|
||||||
|
by_family = {}
|
||||||
|
for r in na:
|
||||||
|
by_family[r["family_id"]] = by_family.get(r["family_id"], 0) + 1
|
||||||
|
by_fold = {}
|
||||||
|
for r in na:
|
||||||
|
by_fold[r["cv_fold"]] = by_fold.get(r["cv_fold"], 0) + 1
|
||||||
|
print(f"family_ids: {len(by_family)} split_groups: {len(set(r['split_group'] for r in na))}")
|
||||||
|
print("folds:", by_fold)
|
||||||
|
|
||||||
|
os.makedirs(OUT_DIR, exist_ok=True)
|
||||||
|
with open(os.path.join(OUT_DIR, "pool.json"), "w") as f:
|
||||||
|
json.dump(na, f, ensure_ascii=False, indent=1)
|
||||||
|
with open(os.path.join(OUT_DIR, "ood.json"), "w") as f:
|
||||||
|
json.dump(ood, f, ensure_ascii=False, indent=1)
|
||||||
|
with open(os.path.join(OUT_DIR, "stats.json"), "w") as f:
|
||||||
|
json.dump({
|
||||||
|
"dev_count": len(dev),
|
||||||
|
"residual_count": len(rows),
|
||||||
|
"non_action_count": len(na),
|
||||||
|
"action_ood_count": len(ood),
|
||||||
|
"routes": by_route,
|
||||||
|
"family_ids": len(by_family),
|
||||||
|
"folds": by_fold,
|
||||||
|
"top_family": dict(sorted(by_family.items(), key=lambda kv: -kv[1])[:15]),
|
||||||
|
}, f, ensure_ascii=False, indent=1)
|
||||||
|
print(f"wrote {OUT_DIR}/{{pool,ood,stats}}.json")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,559 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Slice 23: five-way residual non-action semantic router — corrected population
|
||||||
|
============================================================================
|
||||||
|
|
||||||
|
Slice 22 reported the corpus's fast-path mirror was stale next to the
|
||||||
|
production stage-0 grammars (185 residual rows resolved at runtime). Slice 23
|
||||||
|
derives fast_path_resolved from the real router, rebuilds the corpus, and
|
||||||
|
re-measures the primary slice-22 results on the corrected residual pool.
|
||||||
|
Logic and configs are slice22_main.py verbatim; only OUT_DIR differs.
|
||||||
|
|
||||||
|
Population: the corrected dev-pool residual non-action rows (1509; the pool
|
||||||
|
written by slice23_emit.py). Corrected action rows (720) are OOD probes only.
|
||||||
|
|
||||||
|
Metrics written to /tmp/mvn-s23/results.json:
|
||||||
|
§1 population
|
||||||
|
§2 legacy baseline (legacy.json / legacy_heads.json): acc, macro-F1,
|
||||||
|
per-class P/R/F1, confusion, illegal_action_prediction count
|
||||||
|
§3 e5-linear primary head: C grid, grouped CV OOF, per-fold P/R/F1 +
|
||||||
|
variance + composition
|
||||||
|
§4 floors: majority, centroid (cosine nearest-mean), sparse word+char
|
||||||
|
TF-IDF logistic (slice18 builder), all grouped CV
|
||||||
|
§5 route-family (family_id) leave-family-out
|
||||||
|
§6 knowledge vs memory_write: matched pairs (water/homelab/task) ordering
|
||||||
|
§7 uncertain as an explicit class: P/R/F1 + top confusions
|
||||||
|
§8 OOF confidence: max-softmax correct/wrong, ECE, log-loss, Brier,
|
||||||
|
coverage/accuracy/macro-F1 abstention curves (no threshold chosen)
|
||||||
|
§9 action OOD probes: fold models applied to the corrected action rows
|
||||||
|
§10 artifact cost: head params, serialized bytes, incremental head latency
|
||||||
|
|
||||||
|
No corpus label is changed. No frozen-holdout rows are inspected.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
sys.path.insert(0, HERE)
|
||||||
|
|
||||||
|
import slice18_sparse # noqa: E402
|
||||||
|
import slice19_main # noqa: E402
|
||||||
|
|
||||||
|
EMB_PATH = "/tmp/mvn-experiment/embeddings.json"
|
||||||
|
OUT_DIR = "/tmp/mvn-s23"
|
||||||
|
CLASSES = ["conversation", "knowledge", "memory_write", "system", "uncertain"]
|
||||||
|
CLASS_PREFIX = ["conversation", "knowledge", "memory_write", "system", "uncertain"]
|
||||||
|
C_GRID = [0.1, 1.0, 10.0]
|
||||||
|
|
||||||
|
# Route-family holdouts the report calls out by name (slice-22 brief): every
|
||||||
|
# family that is not part of the shared subject inventory on either side.
|
||||||
|
HOLDOUT_GROUPS = {
|
||||||
|
"capability": ["knowledge:capability-ha", "knowledge:capability-tool"],
|
||||||
|
"world": ["knowledge:world-def", "knowledge:world-explain"],
|
||||||
|
"calendar": ["knowledge:calendar", "knowledge:calendar-time", "knowledge:calendar-next"],
|
||||||
|
"recall": ["knowledge:recall-fact", "knowledge:recall-note", "knowledge:recall-possessive"],
|
||||||
|
"fact": ["fact:meal", "fact:water", "fact:sleep", "fact:shower", "fact:break", "fact:pills", "fact:exercise"],
|
||||||
|
"note": ["note:idea", "note:homelab", "note:task"],
|
||||||
|
"remember": ["free:remember"],
|
||||||
|
"system": None, # all system:*
|
||||||
|
"conversation": None,
|
||||||
|
"uncertain": None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def load_pool_and_embeds():
|
||||||
|
with open(os.path.join(OUT_DIR, "pool.json")) as f:
|
||||||
|
pool = json.load(f)
|
||||||
|
meta, examples = slice18_sparse.load_data()
|
||||||
|
dev = slice18_sparse.filter_dev_pool(examples)
|
||||||
|
by_idx = {e["dev_idx"]: e for e in dev} if "dev_idx" in dev[0] else None
|
||||||
|
# pool rows carry idx = position among dev_pool rows in dev order
|
||||||
|
emb_by_idx = {i: np.asarray(e["embedding"], dtype=np.float64)
|
||||||
|
for i, e in enumerate(dev)}
|
||||||
|
for r in pool:
|
||||||
|
r["emb"] = emb_by_idx[r["idx"]]
|
||||||
|
r["y"] = r["route"]
|
||||||
|
return pool, meta
|
||||||
|
|
||||||
|
|
||||||
|
def oof_proba_grouped(X, y, folds, C=1.0):
|
||||||
|
"""Grouped OOF probability matrix (n×5, class order CLASSES)."""
|
||||||
|
y_idx = np.array([CLASSES.index(c) for c in y])
|
||||||
|
folds = np.asarray(folds)
|
||||||
|
proba = np.zeros((len(y_idx), len(CLASSES)))
|
||||||
|
for te_fold in sorted(set(folds.tolist())):
|
||||||
|
tr = folds != te_fold
|
||||||
|
te = folds == te_fold
|
||||||
|
clf = slice18_sparse.LogisticRegression(
|
||||||
|
C=C, max_iter=2000, solver="lbfgs", random_state=42)
|
||||||
|
clf.fit(X[tr], y_idx[tr])
|
||||||
|
proba[te] = clf.predict_proba(X[te])
|
||||||
|
return proba
|
||||||
|
|
||||||
|
|
||||||
|
def cls_metrics(yt, yp):
|
||||||
|
import sklearn.metrics as m
|
||||||
|
yt = np.asarray(yt)
|
||||||
|
yp = np.asarray(yp)
|
||||||
|
if yt.dtype != np.int64 and yt.dtype != np.int32:
|
||||||
|
yt = np.array([CLASSES.index(c) for c in yt])
|
||||||
|
if yp.dtype != np.int64 and yp.dtype != np.int32:
|
||||||
|
yp = np.array([CLASSES.index(c) for c in yp])
|
||||||
|
labels = list(range(len(CLASSES)))
|
||||||
|
n = len(yt)
|
||||||
|
acc = m.accuracy_score(yt, yp)
|
||||||
|
macro = m.f1_score(yt, yp, average="macro", labels=labels, zero_division=0)
|
||||||
|
pr, rc, f1, sup = m.precision_recall_fscore_support(
|
||||||
|
yt, yp, labels=labels, zero_division=0)
|
||||||
|
per = {c: {"p": float(pr[i]), "r": float(rc[i]), "f1": float(f1[i]), "n": int(sup[i])}
|
||||||
|
for i, c in enumerate(CLASSES)}
|
||||||
|
conf = m.confusion_matrix(yt, yp, labels=labels).tolist()
|
||||||
|
return {"n": n, "acc": acc, "macro_f1": macro, "per_class": per, "confusion": conf}
|
||||||
|
|
||||||
|
|
||||||
|
def fold_report(yt, proba, folds, true_y):
|
||||||
|
out = {}
|
||||||
|
folds_arr = np.asarray(folds)
|
||||||
|
comp = {}
|
||||||
|
for f in sorted(set(folds_arr.tolist())):
|
||||||
|
mask = folds_arr == f
|
||||||
|
yt_f = [CLASSES.index(y) for y in true_y[mask]]
|
||||||
|
comp[f] = {c: int((np.array(true_y[mask]) == c).sum()) for c in CLASSES}
|
||||||
|
per_fold = {}
|
||||||
|
for f in sorted(set(folds_arr.tolist())):
|
||||||
|
mask = folds_arr == f
|
||||||
|
yp = proba[mask].argmax(1).tolist()
|
||||||
|
m = cls_metrics([yt[i] for i in np.where(mask)[0].tolist()], yp)
|
||||||
|
per_fold[f] = {"acc": m["acc"], "macro_f1": m["macro_f1"]}
|
||||||
|
out["composition"] = comp
|
||||||
|
out["per_fold"] = per_fold
|
||||||
|
accs = [v["acc"] for v in per_fold.values()]
|
||||||
|
macros = [v["macro_f1"] for v in per_fold.values()]
|
||||||
|
out["acc_mean"] = float(np.mean(accs))
|
||||||
|
out["acc_std"] = float(np.std(accs))
|
||||||
|
out["macro_f1_mean"] = float(np.mean(macros))
|
||||||
|
out["macro_f1_std"] = float(np.std(macros))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def ece(yt, proba, n_bins=15):
|
||||||
|
conf = proba.max(1)
|
||||||
|
pred = proba.argmax(1)
|
||||||
|
acc = (pred == yt).astype(float)
|
||||||
|
bins = np.linspace(0, 1, n_bins + 1)
|
||||||
|
tot = 0.0
|
||||||
|
details = []
|
||||||
|
counts = 0
|
||||||
|
for i in range(n_bins):
|
||||||
|
lo, hi = bins[i], bins[i + 1]
|
||||||
|
m = (conf >= lo) & (conf < hi) if i < n_bins - 1 else conf >= lo
|
||||||
|
if m.sum() == 0:
|
||||||
|
continue
|
||||||
|
acc_m = acc[m].mean()
|
||||||
|
conf_m = conf[m].mean()
|
||||||
|
w = m.sum() / len(conf)
|
||||||
|
tot += w * abs(acc_m - conf_m)
|
||||||
|
counts += int(m.sum())
|
||||||
|
details.append({"bin": i, "lo": lo, "hi": hi, "conf": float(conf_m),
|
||||||
|
"acc": float(acc_m), "n": int(m.sum())})
|
||||||
|
return {"ece": float(tot), "n_bins": n_bins, "counted": counts, "bins": details}
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
pool, meta = load_pool_and_embeds()
|
||||||
|
pool.sort(key=lambda r: r["idx"])
|
||||||
|
print(f"pool: {len(pool)} rows")
|
||||||
|
|
||||||
|
from sklearn.metrics import brier_score_loss, log_loss
|
||||||
|
|
||||||
|
report = {"population": {}, "legacy": {}, "e5_linear": {}, "floors": {},
|
||||||
|
"family_holdouts": {}, "kmw": {}, "uncertain": {}, "confidence": {},
|
||||||
|
"ood": {}, "artifact": {}}
|
||||||
|
|
||||||
|
# ── §1 population ──────────────────────────────────────────────────────
|
||||||
|
cnt = {}
|
||||||
|
for r in pool:
|
||||||
|
cnt[r["y"]] = cnt.get(r["y"], 0) + 1
|
||||||
|
report["population"] = {
|
||||||
|
"n": len(pool),
|
||||||
|
"routes": cnt,
|
||||||
|
"family_ids": len(set(r["family_id"] for r in pool)),
|
||||||
|
"split_groups": len(set(r["split_group"] for r in pool)),
|
||||||
|
"folds": {str(f): int(sum(1 for r in pool if r["cv_fold"] == f)) for f in sorted(set(r["cv_fold"] for r in pool))},
|
||||||
|
"corpus": {k: v for k, v in meta.items() if k in
|
||||||
|
("dev_count", "residual_count", "fast_path_count",
|
||||||
|
"dimension", "embedder_id", "input_template", "pooling", "normalization")},
|
||||||
|
}
|
||||||
|
print("\n§1 population:", report["population"])
|
||||||
|
|
||||||
|
X = np.vstack([r["emb"] for r in pool])
|
||||||
|
y = np.array([r["y"] for r in pool])
|
||||||
|
folds = np.array([r["cv_fold"] for r in pool])
|
||||||
|
yt = np.array([CLASSES.index(c) for c in y])
|
||||||
|
|
||||||
|
# ── §2 legacy baselines ────────────────────────────────────────────────
|
||||||
|
import collections
|
||||||
|
for tag, fname in [("hash", "legacy.json"), ("heads", "legacy_heads.json")]:
|
||||||
|
path = os.path.join(OUT_DIR, fname)
|
||||||
|
if not os.path.exists(path):
|
||||||
|
continue
|
||||||
|
leg = json.load(open(path))
|
||||||
|
leg_by_idx = {r["idx"]: r for r in leg}
|
||||||
|
yp_leg = []
|
||||||
|
illegal = []
|
||||||
|
for r in pool:
|
||||||
|
lr = leg_by_idx[r["idx"]]
|
||||||
|
if lr["illegal_action_prediction"]:
|
||||||
|
illegal.append(lr)
|
||||||
|
yp_leg.append("action")
|
||||||
|
else:
|
||||||
|
yp_leg.append(lr["class"])
|
||||||
|
yp_leg = np.array(yp_leg)
|
||||||
|
# five-way: an 'action' prediction is an error (outside the label set)
|
||||||
|
yp5 = np.array([("uncertain" if p == "action" else p) for p in yp_leg])
|
||||||
|
m = cls_metrics(y, yp5)
|
||||||
|
m["illegal_action_prediction"] = len(illegal)
|
||||||
|
m["illegal_cases"] = [{"idx": i["idx"], "text": i["text"], "route": i["route"],
|
||||||
|
"intent": i["intent"], "producer": i["producer"],
|
||||||
|
"confidence": i["confidence"]} for i in illegal]
|
||||||
|
# per-cell confusion also shows 'action' column
|
||||||
|
conf_counts = collections.Counter(zip(y, yp_leg))
|
||||||
|
m["confusion_with_action"] = {f"{a}->{b}": int(c) for (a, b), c in conf_counts.items()}
|
||||||
|
report["legacy"][tag] = m
|
||||||
|
print(f"\n§2 legacy ({tag}) acc={m['acc']:.4f} macroF1={m['macro_f1']:.4f} "
|
||||||
|
f"illegal={len(illegal)}")
|
||||||
|
for c in CLASSES:
|
||||||
|
p = m["per_class"][c]
|
||||||
|
print(f" {c:<14} P={p['p']:.3f} R={p['r']:.3f} F1={p['f1']:.3f} n={p['n']}")
|
||||||
|
|
||||||
|
# grammar-pure residual: rows not resolved by any current stage-0 grammar
|
||||||
|
if os.path.exists(os.path.join(OUT_DIR, "legacy.json")):
|
||||||
|
leg = json.load(open(os.path.join(OUT_DIR, "legacy.json")))
|
||||||
|
gh = {r["idx"] for r in leg if r["producer"] == "grammar"}
|
||||||
|
gp_mask = np.array([r["idx"] not in gh for r in pool])
|
||||||
|
report["grammar_drift"] = {
|
||||||
|
"grammar_hits_in_pool": len(gh),
|
||||||
|
"grammar_pure_n": int(gp_mask.sum()),
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── §3 e5-linear primary head ─────────────────────────────────────────
|
||||||
|
print("\n§3 e5-linear")
|
||||||
|
bestC, bestMac = 1.0, -1.0
|
||||||
|
grid = {}
|
||||||
|
for C in C_GRID:
|
||||||
|
p = oof_proba_grouped(X, y, folds, C=C)
|
||||||
|
mp = cls_metrics(y, p.argmax(1).tolist())
|
||||||
|
grid[float(C)] = {"acc": mp["acc"], "macro_f1": mp["macro_f1"]}
|
||||||
|
print(f" C={C} acc={mp['acc']:.4f} macroF1={mp['macro_f1']:.4f}")
|
||||||
|
if mp["macro_f1"] > bestMac:
|
||||||
|
bestMac, bestC = mp["macro_f1"], C
|
||||||
|
print(f" -> best C={bestC}")
|
||||||
|
p_best = oof_proba_grouped(X, y, folds, C=bestC)
|
||||||
|
m_best = cls_metrics(y, p_best.argmax(1).tolist())
|
||||||
|
m_best["C"] = bestC
|
||||||
|
m_best["C_grid"] = grid
|
||||||
|
m_best["folds"] = fold_report(yt, p_best, folds, y)
|
||||||
|
report["e5_linear"] = m_best
|
||||||
|
for f, v in m_best["folds"]["per_fold"].items():
|
||||||
|
print(f" fold {f}: acc={v['acc']:.4f} macroF1={v['macro_f1']:.4f}")
|
||||||
|
print(f" fold acc mean={m_best['folds']['acc_mean']:.4f} "
|
||||||
|
f"std={m_best['folds']['acc_std']:.4f}; "
|
||||||
|
f"macroF1 mean={m_best['folds']['macro_f1_mean']:.4f} "
|
||||||
|
f"std={m_best['folds']['macro_f1_std']:.4f}")
|
||||||
|
for c in CLASSES:
|
||||||
|
p_ = m_best["per_class"][c]
|
||||||
|
print(f" {c:<14} P={p_['p']:.3f} R={p_['r']:.3f} F1={p_['f1']:.3f} n={p_['n']}")
|
||||||
|
|
||||||
|
# grammar-pure sensitivity for the primary head
|
||||||
|
if "grammar_drift" in report:
|
||||||
|
mp_gp = cls_metrics(y[gp_mask], p_best[gp_mask].argmax(1).tolist())
|
||||||
|
report["e5_linear"]["grammar_pure"] = {
|
||||||
|
"acc": mp_gp["acc"], "macro_f1": mp_gp["macro_f1"], "n": int(gp_mask.sum())}
|
||||||
|
|
||||||
|
# ── §4 floors ─────────────────────────────────────────────────────────
|
||||||
|
print("\n§4 floors")
|
||||||
|
# majority floor
|
||||||
|
maj = CLASSES.index("knowledge")
|
||||||
|
ym = np.full(len(y), maj)
|
||||||
|
mm = cls_metrics(y, ym)
|
||||||
|
report["floors"]["majority"] = {"acc": mm["acc"], "macro_f1": mm["macro_f1"],
|
||||||
|
"per_class": mm["per_class"]}
|
||||||
|
print(f" majority (predict {CLASSES[maj]}): acc={mm['acc']:.4f} macroF1={mm['macro_f1']:.4f}")
|
||||||
|
|
||||||
|
# centroid floor: cosine to per-class mean of the training folds' embeddings
|
||||||
|
cf_proba = np.zeros((len(yt), len(CLASSES)))
|
||||||
|
folds_arr = np.asarray(folds)
|
||||||
|
for te_fold in sorted(set(folds_arr.tolist())):
|
||||||
|
tr = folds_arr != te_fold
|
||||||
|
te = folds_arr == te_fold
|
||||||
|
centroids = []
|
||||||
|
for c in CLASSES:
|
||||||
|
idxs = np.where(tr & (y == c))[0]
|
||||||
|
ctr = X[idxs].mean(axis=0)
|
||||||
|
ctr = ctr / np.linalg.norm(ctr)
|
||||||
|
centroids.append(ctr)
|
||||||
|
Cm = np.vstack(centroids)
|
||||||
|
sims = X[te] @ Cm.T
|
||||||
|
cf_proba[te] = sims
|
||||||
|
yc = cf_proba.argmax(1)
|
||||||
|
# accuracy + macroF1 with the same 5-way
|
||||||
|
mc = cls_metrics(y, yc.tolist())
|
||||||
|
report["floors"]["centroid"] = {"acc": mc["acc"], "macro_f1": mc["macro_f1"],
|
||||||
|
"per_class": mc["per_class"]}
|
||||||
|
print(f" centroid cosine: acc={mc['acc']:.4f} macroF1={mc['macro_f1']:.4f}")
|
||||||
|
|
||||||
|
# sparse word+char logistic (slice18 builder, grouped CV, five-way)
|
||||||
|
texts = [r["n_text"] for r in pool]
|
||||||
|
Xs, _vec = slice18_sparse.build_features(texts, "both")
|
||||||
|
psp = np.zeros((len(yt), len(CLASSES)))
|
||||||
|
for te_fold in sorted(set(folds_arr.tolist())):
|
||||||
|
tr = folds_arr != te_fold
|
||||||
|
te = folds_arr == te_fold
|
||||||
|
clf = slice18_sparse.LogisticRegression(
|
||||||
|
C=1.0, max_iter=2000, solver="lbfgs", random_state=42)
|
||||||
|
clf.fit(Xs[tr], yt[tr])
|
||||||
|
psp[te] = clf.predict_proba(Xs[te])
|
||||||
|
msp = cls_metrics(y, psp.argmax(1).tolist())
|
||||||
|
report["floors"]["sparse_word_char"] = {
|
||||||
|
"acc": msp["acc"], "macro_f1": msp["macro_f1"], "per_class": msp["per_class"],
|
||||||
|
"vocab": slice18_sparse.vocab_size(_vec)}
|
||||||
|
print(f" sparse both: acc={msp['acc']:.4f} macroF1={msp['macro_f1']:.4f} "
|
||||||
|
f"vocab={report['floors']['sparse_word_char']['vocab']}")
|
||||||
|
|
||||||
|
# ── §5 route-family holdouts ─────────────────────────────────────────
|
||||||
|
print("\n§5 route-family holdouts")
|
||||||
|
fam = np.array([r["family_id"] for r in pool])
|
||||||
|
holdouts = {}
|
||||||
|
all_fams = sorted(set(fam.tolist()))
|
||||||
|
for grp, fams in HOLDOUT_GROUPS.items():
|
||||||
|
if fams is None:
|
||||||
|
fams = [f for f in all_fams if f.startswith(grp + ":")]
|
||||||
|
mask = np.isin(fam, fams)
|
||||||
|
if mask.sum() == 0:
|
||||||
|
continue
|
||||||
|
tr = ~mask
|
||||||
|
clf = slice18_sparse.LogisticRegression(
|
||||||
|
C=bestC, max_iter=2000, solver="lbfgs", random_state=42)
|
||||||
|
clf.fit(X[tr], yt[tr])
|
||||||
|
ypgrp = clf.predict(X[mask])
|
||||||
|
m = cls_metrics([CLASSES.index(c) for c in y[mask]], ypgrp.tolist())
|
||||||
|
m["families"] = fams
|
||||||
|
m["rows"] = int(mask.sum())
|
||||||
|
holdouts[grp] = {"acc": m["acc"], "macro_f1": m["macro_f1"], "n": int(mask.sum()),
|
||||||
|
"per_class": m["per_class"]}
|
||||||
|
print(f" {grp:<14} n={m['rows']} acc={m['acc']:.4f} macroF1={m['macro_f1']:.4f}")
|
||||||
|
# full leave-one-family-out summary
|
||||||
|
lofo_accs = []
|
||||||
|
lofo_f1s = []
|
||||||
|
for f in all_fams:
|
||||||
|
mask = fam == f
|
||||||
|
tr = ~mask
|
||||||
|
clf = slice18_sparse.LogisticRegression(
|
||||||
|
C=bestC, max_iter=2000, solver="lbfgs", random_state=42)
|
||||||
|
clf.fit(X[tr], yt[tr])
|
||||||
|
ypf = clf.predict(X[mask])
|
||||||
|
m = cls_metrics([CLASSES.index(c) for c in y[mask]], ypf.tolist())
|
||||||
|
lofo_accs.append(m["acc"])
|
||||||
|
lofo_f1s.append(m["macro_f1"])
|
||||||
|
holdouts["_all_49_lo_"] = {"n_families": len(all_fams),
|
||||||
|
"acc_mean": float(np.mean(lofo_accs)),
|
||||||
|
"macro_f1_mean": float(np.mean(lofo_f1s))}
|
||||||
|
report["family_holdouts"] = holdouts
|
||||||
|
print(f" leave-one-family-out over {len(all_fams)} families: "
|
||||||
|
f"acc mean={np.mean(lofo_accs):.4f} macroF1 mean={np.mean(lofo_f1s):.4f}")
|
||||||
|
|
||||||
|
# ── §6 knowledge vs memory_write ─────────────────────────────────────
|
||||||
|
print("\n§6 knowledge vs memory_write")
|
||||||
|
# reuse e5-linear OOF: does the model put the higher probability on the
|
||||||
|
# right side (memory_write for a write, knowledge for a recall)?
|
||||||
|
conf_km = np.zeros((2, 2))
|
||||||
|
pk = p_best[:, CLASSES.index("knowledge")]
|
||||||
|
pmw = p_best[:, CLASSES.index("memory_write")]
|
||||||
|
for i in range(len(yt)):
|
||||||
|
t = y[i]
|
||||||
|
if t == "knowledge":
|
||||||
|
conf_km[0, 1 if pmw[i] > pk[i] else 0] += 1
|
||||||
|
elif t == "memory_write":
|
||||||
|
conf_km[1, 1 if pmw[i] >= pk[i] else 0] += 1
|
||||||
|
report["kmw"] = {"confusion_p_ordered": conf_km.tolist()}
|
||||||
|
|
||||||
|
# matched pairs with shared subject lexemes, corpus-justified
|
||||||
|
def build_pairs(subject, fam_k, fam_mw):
|
||||||
|
kr = [r for r in pool if r["family_id"] in fam_k]
|
||||||
|
mr = [r for r in pool if r["family_id"] in fam_mw]
|
||||||
|
pairs = []
|
||||||
|
for mw in mr:
|
||||||
|
for k in kr:
|
||||||
|
if subject in mw["n_text"] and subject in k["n_text"]:
|
||||||
|
pairs.append((mw["idx"], k["idx"], mw["n_text"], k["n_text"]))
|
||||||
|
return pairs
|
||||||
|
|
||||||
|
sets = {
|
||||||
|
"water": build_pairs("вод", ["knowledge:recall-fact"], ["fact:water"]),
|
||||||
|
"homelab": build_pairs("dns", ["knowledge:homelab-status"], ["note:homelab"])
|
||||||
|
+ build_pairs("сервер", ["knowledge:homelab-status"], ["note:homelab"])
|
||||||
|
+ build_pairs("vlan", ["knowledge:homelab-status"], ["note:homelab"]),
|
||||||
|
"task": build_pairs("задач", ["knowledge:task-check", "knowledge:deadline"],
|
||||||
|
["note:task"]),
|
||||||
|
}
|
||||||
|
idx_of = {r["idx"]: i for i, r in enumerate(pool)}
|
||||||
|
pair_rep = {}
|
||||||
|
for name, pairs in sets.items():
|
||||||
|
if not pairs:
|
||||||
|
continue
|
||||||
|
ok = 0
|
||||||
|
margins = []
|
||||||
|
bad = []
|
||||||
|
for mi, ki, mx, kx in pairs:
|
||||||
|
mi_i, ki_i = idx_of[mi], idx_of[ki]
|
||||||
|
# MW row should get a higher memory_write probability than the K row
|
||||||
|
mk = (pmw[mi_i] + 0.0)
|
||||||
|
if pmw[mi_i] > pmw[ki_i]:
|
||||||
|
ok += 1
|
||||||
|
else:
|
||||||
|
bad.append((mx[:46], round(float(pmw[mi_i]), 3), kx[:46], round(float(pmw[ki_i]), 3)))
|
||||||
|
margins.append(pmw[mi_i] - pmw[ki_i])
|
||||||
|
pair_rep[name] = {
|
||||||
|
"pairs": len(pairs),
|
||||||
|
"mw_over_k_order_acc": ok / len(pairs),
|
||||||
|
"mean_margin": float(np.mean(margins)),
|
||||||
|
"reversed_examples": bad[:6],
|
||||||
|
}
|
||||||
|
print(f" {name}: pairs={len(pairs)} order_acc={ok/len(pairs):.3f} "
|
||||||
|
f"mean_margin={np.mean(margins):+.3f}")
|
||||||
|
report["kmw"]["matched_pairs"] = pair_rep
|
||||||
|
|
||||||
|
# ── §7 uncertain as explicit class ────────────────────────────────────
|
||||||
|
print("\n§7 uncertain")
|
||||||
|
up = m_best["per_class"]["uncertain"]
|
||||||
|
uc = m_best["confusion"][CLASSES.index("uncertain")]
|
||||||
|
report["uncertain"] = {
|
||||||
|
"per_class": up,
|
||||||
|
"row_from_uncertain": {CLASSES[j]: int(uc[j]) for j in range(5)},
|
||||||
|
"row_to_uncertain": {CLASSES[j]: int(m_best["confusion"][j][CLASSES.index("uncertain")])
|
||||||
|
for j in range(5)},
|
||||||
|
}
|
||||||
|
print(f" uncertain n={up['n']} P={up['p']:.3f} R={up['r']:.3f} F1={up['f1']:.3f}")
|
||||||
|
print(" wrong-→label pulled from uncertain:", report["uncertain"]["row_from_uncertain"])
|
||||||
|
print(" →uncertain pulled from:", report["uncertain"]["row_to_uncertain"])
|
||||||
|
|
||||||
|
# ── §8 OOF confidence / calibration / abstention ─────────────────────
|
||||||
|
print("\n§8 confidence / calibration")
|
||||||
|
conf = p_best.max(1)
|
||||||
|
right = (p_best.argmax(1) == yt)
|
||||||
|
cer = {
|
||||||
|
"correct_conf_mean": float(conf[right].mean()),
|
||||||
|
"correct_conf_median": float(np.median(conf[right])),
|
||||||
|
"wrong_conf_mean": float(conf[~right].mean()),
|
||||||
|
"wrong_conf_median": float(np.median(conf[~right])),
|
||||||
|
"ece": ece(yt, p_best)["ece"],
|
||||||
|
"ece_bins": ece(yt, p_best)["bins"],
|
||||||
|
"log_loss": float(log_loss(yt, p_best, labels=[0, 1, 2, 3, 4])),
|
||||||
|
}
|
||||||
|
# Brier is label-set specific: one-vs-rest mean
|
||||||
|
briers = []
|
||||||
|
for i in range(5):
|
||||||
|
briers.append(brier_score_loss((yt == i).astype(int), p_best[:, i]))
|
||||||
|
cer["brier_macro"] = float(np.mean(briers))
|
||||||
|
report["confidence"] = cer
|
||||||
|
print(f" right conf mean={cer['correct_conf_mean']:.3f} "
|
||||||
|
f"wrong conf mean={cer['wrong_conf_mean']:.3f} ECE={cer['ece']:.4f}")
|
||||||
|
print(f" log_loss={cer['log_loss']:.4f} brier_macro={cer['brier_macro']:.4f}")
|
||||||
|
|
||||||
|
thr_grid = np.linspace(0.10, 0.98, 45)
|
||||||
|
abst = []
|
||||||
|
for t in thr_grid:
|
||||||
|
cov = (conf >= t).mean()
|
||||||
|
if cov == 0:
|
||||||
|
continue
|
||||||
|
keep = conf >= t
|
||||||
|
yt_k = yt[keep]
|
||||||
|
yp_k = p_best[keep].argmax(1)
|
||||||
|
mk_ = cls_metrics(yt_k.tolist(), yp_k.tolist())
|
||||||
|
abst.append({"threshold": round(float(t), 3), "coverage": float(cov),
|
||||||
|
"accuracy": mk_["acc"], "macro_f1": mk_["macro_f1"]})
|
||||||
|
report["confidence"]["abstention_curve"] = abst
|
||||||
|
print(" threshold | coverage | accuracy | macroF1 (first 6/45 + knee)")
|
||||||
|
for row in abst[::9]:
|
||||||
|
print(f" {row['threshold']:.2f} | {row['coverage']:.3f} | "
|
||||||
|
f"{row['accuracy']:.3f} | {row['macro_f1']:.3f}")
|
||||||
|
|
||||||
|
# ── §9 action OOD probes ──────────────────────────────────────────────
|
||||||
|
print("\n§9 action OOD")
|
||||||
|
ood_rows = [r for r in json.load(open(os.path.join(OUT_DIR, "ood.json")))]
|
||||||
|
emb_by_idx = {i: np.asarray(e["embedding"], dtype=np.float64)
|
||||||
|
for i, e in enumerate(slice18_sparse.filter_dev_pool(
|
||||||
|
slice18_sparse.load_data()[1]))}
|
||||||
|
Xo = np.vstack([emb_by_idx[r["idx"]] for r in ood_rows])
|
||||||
|
fold_models = []
|
||||||
|
for te_fold in sorted(set(folds_arr.tolist())):
|
||||||
|
tr = folds_arr != te_fold
|
||||||
|
clf = slice18_sparse.LogisticRegression(
|
||||||
|
C=bestC, max_iter=2000, solver="lbfgs", random_state=42)
|
||||||
|
clf.fit(X[tr], yt[tr])
|
||||||
|
fold_models.append(clf)
|
||||||
|
# XX
|
||||||
|
# comparable to the non-action in-fold behaviour
|
||||||
|
po = np.zeros((len(Xo), 5))
|
||||||
|
for clf in fold_models:
|
||||||
|
po += clf.predict_proba(Xo)
|
||||||
|
po /= len(fold_models)
|
||||||
|
ood_top = int(np.argmax(po.mean(0)))
|
||||||
|
ood_conf = po.max(1)
|
||||||
|
ood_pred = po.argmax(1)
|
||||||
|
top_dist = {CLASSES[i]: int((ood_pred == i).sum()) for i in range(5)}
|
||||||
|
confident_na = int((ood_conf > 0.9).sum())
|
||||||
|
report["ood"] = {
|
||||||
|
"n": len(ood_rows),
|
||||||
|
"top_class": CLASSES[int(ood_top)],
|
||||||
|
"top_class_dist": top_dist,
|
||||||
|
"conf_gt_0.9": confident_na,
|
||||||
|
"conf_gt_0.9_frac": float(confident_na / len(ood_rows)),
|
||||||
|
"conf_mean": float(ood_conf.mean()),
|
||||||
|
"conf_median": float(np.median(ood_conf)),
|
||||||
|
}
|
||||||
|
print(f" action OOD n={len(ood_rows)}: most-confident class={report['ood']['top_class']} "
|
||||||
|
f"dist={top_dist}")
|
||||||
|
print(f" conf>0.9: {confident_na} ({confident_na/len(ood_rows):.3f}) "
|
||||||
|
f"conf mean={report['ood']['conf_mean']:.3f}")
|
||||||
|
|
||||||
|
# ── §10 artifact cost ────────────────────────────────────────────────
|
||||||
|
print("\n§10 artifact")
|
||||||
|
n_params = len(CLASSES) * X.shape[1] + len(CLASSES)
|
||||||
|
fp32 = n_params * 4
|
||||||
|
report["artifact"] = {
|
||||||
|
"e5_dim": X.shape[1],
|
||||||
|
"head_params": n_params,
|
||||||
|
"head_fp32_bytes": fp32,
|
||||||
|
"head_fp32_kib": fp32 / 1024,
|
||||||
|
"head_int8_bytes": n_params,
|
||||||
|
}
|
||||||
|
# incremental latency of the linear head over a batch of 1 (µs)
|
||||||
|
clf = slice18_sparse.LogisticRegression(C=bestC, max_iter=2000, solver="lbfgs", random_state=42)
|
||||||
|
clf.fit(X, yt)
|
||||||
|
x1 = X[:1]
|
||||||
|
for _ in range(50):
|
||||||
|
clf.predict_proba(x1)
|
||||||
|
lat = []
|
||||||
|
for _ in range(2000):
|
||||||
|
t0 = time.perf_counter_ns()
|
||||||
|
clf.predict_proba(x1)
|
||||||
|
lat.append((time.perf_counter_ns() - t0) / 1e3)
|
||||||
|
lat = np.array(lat)
|
||||||
|
report["artifact"]["head_latency_us_mean"] = float(lat.mean())
|
||||||
|
report["artifact"]["head_latency_us_p50"] = float(np.median(lat))
|
||||||
|
print(f" head params={n_params} fp32={fp32/1024:.2f}KiB "
|
||||||
|
f"lat mean={lat.mean():.2f}us p50={np.median(lat):.2f}us")
|
||||||
|
|
||||||
|
with open(os.path.join(OUT_DIR, "results.json"), "w") as f:
|
||||||
|
json.dump(report, f, ensure_ascii=False, indent=1, default=float)
|
||||||
|
print(f"\nwrote {OUT_DIR}/results.json")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
+7403
-7781
File diff suppressed because it is too large
Load Diff
@@ -142,7 +142,7 @@ func TestLegacyBaseline(t *testing.T) {
|
|||||||
func buildMinimalRouter(t *testing.T) LegacyRouter {
|
func buildMinimalRouter(t *testing.T) LegacyRouter {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
// Use the same act matcher and seed loading as the eval package.
|
// Use the same act matcher and seed loading as the eval package.
|
||||||
acts := router.DefaultActMatcher{Fns: actVerbList()}
|
acts := router.DefaultActMatcher{Fns: ExperimentActVerbs()}
|
||||||
cls := buildSeededClassifier(t, router.NewHashEmbedder(1024))
|
cls := buildSeededClassifier(t, router.NewHashEmbedder(1024))
|
||||||
r := router.New(router.Config{
|
r := router.New(router.Config{
|
||||||
Grammars: router.StageZeroGrammars(acts),
|
Grammars: router.StageZeroGrammars(acts),
|
||||||
|
|||||||
@@ -0,0 +1,82 @@
|
|||||||
|
package semantic
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/kami/maven/internal/router"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ExperimentActVerbs is the act allowlist shared by the experiment's legacy
|
||||||
|
// baseline (the eval fixture and the slice-22 harness), the routed-heads
|
||||||
|
// harness, and the corpus fast-path derivation. One non-test list so the
|
||||||
|
// corpus derivation can never drift from what the measurement router accepts.
|
||||||
|
func ExperimentActVerbs() []string {
|
||||||
|
return []string{
|
||||||
|
"перезапусти", "перезагрузи", "выключи", "включи", "останови", "запусти",
|
||||||
|
"закрой", "открой", "сделай", "поставь",
|
||||||
|
"restart", "reboot", "stop", "start", "turn off", "turn on", "open", "close",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func experimentActMatcher() router.ActMatcher {
|
||||||
|
return router.DefaultActMatcher{Fns: ExperimentActVerbs()}
|
||||||
|
}
|
||||||
|
|
||||||
|
// FastPathOutcome is the derived fast-path classification for one surface.
|
||||||
|
// Matched mirrors TryFastPath; Grammar names the winning stage-0 grammar for
|
||||||
|
// attribution, or the empty string when nothing resolved.
|
||||||
|
type FastPathOutcome struct {
|
||||||
|
Matched bool
|
||||||
|
Grammar string
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
fastPathOnce sync.Once
|
||||||
|
fastPathRouter *router.Router
|
||||||
|
fastPathGrammars []router.Grammar
|
||||||
|
)
|
||||||
|
|
||||||
|
// DeriveFastPath mirrors the production fast path for a surface: it runs
|
||||||
|
// TryFastPath over the daemon's ordered stage-0 grammar list with the
|
||||||
|
// experiment's act allowlist — the exact router the legacy baseline measures —
|
||||||
|
// and reports whether a grammar resolved the utterance. This is the
|
||||||
|
// authoritative derivation for corpus fast_path_resolved metadata. The
|
||||||
|
// classifier, threshold and LLM never run on the fast path, so they are not
|
||||||
|
// wired here.
|
||||||
|
func DeriveFastPath(text string) FastPathOutcome {
|
||||||
|
fastPathOnce.Do(func() {
|
||||||
|
acts := experimentActMatcher()
|
||||||
|
fastPathGrammars = router.StageZeroGrammars(acts)
|
||||||
|
fastPathRouter = router.New(router.Config{
|
||||||
|
Grammars: fastPathGrammars,
|
||||||
|
Extractor: router.Extractor{
|
||||||
|
Time: router.StubDateTimeParser{},
|
||||||
|
Acts: acts,
|
||||||
|
Facts: router.DefaultFactParser{},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
res, err := fastPathRouter.TryFastPath(context.Background(), router.NormalizedInput{
|
||||||
|
Text: text,
|
||||||
|
MatchText: router.NormalizeMatchText(text),
|
||||||
|
}, time.Now())
|
||||||
|
if err != nil || !res.Matched {
|
||||||
|
return FastPathOutcome{}
|
||||||
|
}
|
||||||
|
// Attribute the winner by replaying TryFastPath's ordered first-accept
|
||||||
|
// walk, including the wake-stripped alternate. A matcher that matched the
|
||||||
|
// shape but declined the content falls through, exactly as the router does.
|
||||||
|
stripped, hadWake := router.StripWakeToken(text)
|
||||||
|
for _, g := range fastPathGrammars {
|
||||||
|
_, matched, ok := g.Evaluate(text)
|
||||||
|
if !matched && hadWake {
|
||||||
|
_, matched, ok = g.Evaluate(stripped)
|
||||||
|
}
|
||||||
|
if matched && ok {
|
||||||
|
return FastPathOutcome{Matched: true, Grammar: g.Name}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return FastPathOutcome{Matched: true}
|
||||||
|
}
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
package semantic
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
// TestFastPathDerivationInvariant asserts that every development-pool row's
|
||||||
|
// stored fast-path flag exactly matches what the production fast path derives
|
||||||
|
// today (DeriveFastPath runs TryFastPath over the stage-0 grammars with the
|
||||||
|
// experiment's act allowlist). Frozen holdout rows are preserved verbatim
|
||||||
|
// across merges and are exempt — their drift is reported, never silently
|
||||||
|
// rewritten. A stale development value is a corpus-build error.
|
||||||
|
func TestFastPathDerivationInvariant(t *testing.T) {
|
||||||
|
exs, err := LoadCorpus()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load corpus: %v", err)
|
||||||
|
}
|
||||||
|
_, dev, _ := FrozenHoldoutSplit(exs)
|
||||||
|
devSet := make(map[string]bool, len(dev))
|
||||||
|
for _, e := range dev {
|
||||||
|
devSet[e.SourceID] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
stale := 0
|
||||||
|
for _, e := range exs {
|
||||||
|
if !devSet[e.SourceID] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if got := DeriveFastPath(e.Text).Matched; got != e.FastPathResolved {
|
||||||
|
t.Errorf("dev row %s (route=%s) fast_path_resolved=%v but router derives %v: %q",
|
||||||
|
e.SourceID, e.Route, e.FastPathResolved, got, e.Text)
|
||||||
|
stale++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if stale > 0 {
|
||||||
|
t.Fatalf("%d development rows have stale fast-path metadata", stale)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -19,14 +19,6 @@ var seedIntents = []router.Intent{
|
|||||||
router.IntentNote, router.IntentQuery, router.IntentChat, router.IntentSystem,
|
router.IntentNote, router.IntentQuery, router.IntentChat, router.IntentSystem,
|
||||||
}
|
}
|
||||||
|
|
||||||
func actVerbList() []string {
|
|
||||||
return []string{
|
|
||||||
"перезапусти", "перезагрузи", "выключи", "включи", "останови", "запусти",
|
|
||||||
"закрой", "открой", "сделай", "поставь",
|
|
||||||
"restart", "reboot", "stop", "start", "turn off", "turn on", "open", "close",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildSeededClassifier(t *testing.T, emb router.Embedder) *router.Classifier {
|
func buildSeededClassifier(t *testing.T, emb router.Embedder) *router.Classifier {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
cls := router.NewClassifier(emb)
|
cls := router.NewClassifier(emb)
|
||||||
|
|||||||
Reference in New Issue
Block a user