router/semantic: slice 23 corpus fast-path reconciliation — DeriveFastPath over the real router replaces the regex mirror, factory/merge validated, corpus rebuilt (dataset_hash unchanged), drift diagnostic and rerun tooling
This commit is contained in:
@@ -15,7 +15,6 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -1089,85 +1088,11 @@ func generatorQuestion(seed semantic.SemanticSeed) []semantic.GeneratedSurface {
|
||||
}
|
||||
|
||||
// ─── Fast-path classifier ────────────────────────────────────────────────────
|
||||
// Simplified pattern matcher that mirrors stage-0 grammar outcomes.
|
||||
|
||||
var (
|
||||
reminderVerbRe = regexp.MustCompile(`(?i)^\s*(напомни|напомните|напомнить|напоминай|разбуди|разбудите|разбудить|буди|remind|wake)\b`)
|
||||
timeQueryRe = regexp.MustCompile(`(?i)^\s*(сколько\s+сейчас\s+времени|который\s+час|какое\s+число|какой\s+день|what\s+time|what's\s+the\s+date)\b`)
|
||||
captureVerbRe = regexp.MustCompile(`(?i)^\s*(запиши|запомни|отметь|заметь|добавь|сохрани|занеси|внеси|note|remember|log|save|add)\b`)
|
||||
praxisItemRe = regexp.MustCompile(`(?i)\b(item[_-][a-z0-9_-]+)`)
|
||||
quietOnRe = regexp.MustCompile(`(?i)(тихий\s+режим|режим\s+тишина|тише|потише|quiet\s+(mode|on))`)
|
||||
quietOffRe = regexp.MustCompile(`(?i)(хватит\s+тихого|громкий\s+режим|выключи\s+тихий|quiet\s+(off|end))`)
|
||||
taskDoneRe = regexp.MustCompile(`(?i)^\s*(закрой\s+задачу|заверши\s+задачу|close\s+the\s+task|finish\s+the\s+task)\b`)
|
||||
taskDropRe = regexp.MustCompile(`(?i)^\s*(убери\s+задачу|удали\s+задачу|отмени\s+задачу|drop\s+the\s+task|remove\s+the\s+task)\b`)
|
||||
)
|
||||
|
||||
// tool aliases → fast-path matchers (subset of what DefaultActMatcher would match)
|
||||
var toolAliasRe = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?i)^\s*(перезапусти|перезагрузи|рестарт)\b`),
|
||||
regexp.MustCompile(`(?i)^\s*(останови|выключи)\b`),
|
||||
regexp.MustCompile(`(?i)^\s*(запусти|старт)\b`),
|
||||
regexp.MustCompile(`(?i)^\s*(перезапусти\s+контейнер|перезагрузи\s+контейнер)\b`),
|
||||
regexp.MustCompile(`(?i)^\s*(останови\s+контейнер)\b`),
|
||||
regexp.MustCompile(`(?i)^\s*(перезагрузи\s+сервер|перезагрузи\s+хост|ребут)\b`),
|
||||
regexp.MustCompile(`(?i)^\s*(покажи\s+статус|проверь\s+статус|статус)\b`),
|
||||
regexp.MustCompile(`(?i)^\s*(покажи\s+контейнеры|список\s+контейнеров|что\s+запущено)\b`),
|
||||
regexp.MustCompile(`(?i)^\s*(покажи\s+uptime|аптайм|как\s+работает\s+сервер)\b`),
|
||||
regexp.MustCompile(`(?i)^\s*(покажи\s+диск|сколько\s+места\s+на\s+диске)\b`),
|
||||
regexp.MustCompile(`(?i)^\s*(покажи\s+память|свободная\s+память)\b`),
|
||||
regexp.MustCompile(`(?i)^\s*(покажи\s+логи|логи|лог)\b`),
|
||||
}
|
||||
|
||||
// praxis lifecycle verbs (bare, without item reference → residual action)
|
||||
var praxisLifecycleRe = regexp.MustCompile(`(?i)^\s*(закрой|закрыть|resolve|close|принято|принять|acknowledge|ack|игнорируй|игнорировать|пропусти|ignore|skip|закрепи|закрепить|pin)\b`)
|
||||
var praxisAttentionRe = regexp.MustCompile(`(?i)^\s*(что\s+требует\s+внимания|что\s+нового\s+по\s+(задачам|проектам|сервисам)|needs\s+attention)\b`)
|
||||
var praxisChangesRe = regexp.MustCompile(`(?i)^\s*(что\s+изменилось|какие\s+изменения|changed|changes)\b`)
|
||||
var praxisEntityRe = regexp.MustCompile(`(?i)^\s*(статус|status|как\s+дела\s+у|how\s+is)\s+`)
|
||||
|
||||
// classifyFastPath determines whether a surface would be handled by stage-0.
|
||||
func classifyFastPath(text string) bool {
|
||||
t := strings.TrimSpace(text)
|
||||
|
||||
// Reminder verbs → always fast-path
|
||||
if reminderVerbRe.MatchString(t) {
|
||||
return true
|
||||
}
|
||||
// Time/date queries → fast-path
|
||||
if timeQueryRe.MatchString(t) {
|
||||
return true
|
||||
}
|
||||
// Task-status commands → fast-path
|
||||
if taskDoneRe.MatchString(t) || taskDropRe.MatchString(t) {
|
||||
return true
|
||||
}
|
||||
// Tool aliases → fast-path (only for verbs alone, not when combined with objects in ways the matcher wouldn't handle)
|
||||
for _, re := range toolAliasRe {
|
||||
if re.MatchString(t) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// Praxis lifecycle with item reference → fast-path
|
||||
if praxisLifecycleRe.MatchString(t) && praxisItemRe.MatchString(t) {
|
||||
return true
|
||||
}
|
||||
// Praxis attention/changes → fast-path
|
||||
if praxisAttentionRe.MatchString(t) || praxisChangesRe.MatchString(t) {
|
||||
return true
|
||||
}
|
||||
// Praxis entity attention → fast-path
|
||||
if praxisEntityRe.MatchString(t) {
|
||||
return true
|
||||
}
|
||||
// Quiet mode → fast-path (resolved pre-route)
|
||||
if quietOnRe.MatchString(t) || quietOffRe.MatchString(t) {
|
||||
return true
|
||||
}
|
||||
// Capture verbs → some are fast-path (note capture)
|
||||
if captureVerbRe.MatchString(t) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
// Fast-path metadata is derived from the real router, not a regex mirror:
|
||||
// DeriveFastPath runs the production TryFastPath over the stage-0 grammars
|
||||
// with the experiment's act allowlist (see internal/router/semantic/fastpath.go).
|
||||
// A hand-maintained pattern list drifted from the grammars and labelled 185
|
||||
// residual rows as fast (docs/evals/2026-09-08-slice22-*); it is removed.
|
||||
|
||||
// ─── Helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -1241,7 +1166,7 @@ func buildCorpus(seeds []semantic.SemanticSeed) []semantic.RouteExample {
|
||||
exampleCounter++
|
||||
srcID := fmt.Sprintf("%s-%03d", seed.ID, exampleCounter)
|
||||
|
||||
fp := classifyFastPath(surf.Text)
|
||||
fp := semantic.DeriveFastPath(surf.Text).Matched
|
||||
isResidual := !fp
|
||||
|
||||
tags := append([]string{}, seed.Tags...)
|
||||
|
||||
@@ -92,13 +92,37 @@ func main() {
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "merged: %d examples (skipped %d duplicates)\n", len(merged), skipped)
|
||||
|
||||
// 5. Validate
|
||||
// 5. Fast-path metadata must match the real router. V2 rows are derived
|
||||
// by interactively-labeled construction; a stale manually-supplied value
|
||||
// is a build error, never silently rewritten. Frozen holdout rows are
|
||||
// preserved verbatim — report drift, do not touch them.
|
||||
v2Checked, frozenChecked, frozenDrift := 0, 0, 0
|
||||
for _, e := range merged {
|
||||
derived := semantic.DeriveFastPath(e.Text).Matched
|
||||
if frozenTexts[strings.TrimSpace(e.Text)] {
|
||||
frozenChecked++
|
||||
if derived != e.FastPathResolved {
|
||||
frozenDrift++
|
||||
fmt.Fprintf(os.Stderr, "frozen drift: %q stored=%v derived=%v\n", e.Text, e.FastPathResolved, derived)
|
||||
}
|
||||
continue
|
||||
}
|
||||
v2Checked++
|
||||
if derived != e.FastPathResolved {
|
||||
log.Fatalf("v2 row (source=%s source_id=%s) fast_path_resolved=%v but router derives %v: %q",
|
||||
e.Source, e.SourceID, e.FastPathResolved, derived, e.Text)
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "fast-path check: v2 rows %d OK, frozen rows %d (drift %d, reported only)\n",
|
||||
v2Checked, frozenChecked, frozenDrift)
|
||||
|
||||
// 6. Validate
|
||||
if err := semantic.ValidateCorpus(merged); err != nil {
|
||||
log.Fatalf("validation failed: %v", err)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "validation: OK\n")
|
||||
|
||||
// 6. Compute dataset hash
|
||||
// 7. Compute dataset hash
|
||||
texts := make([]string, len(merged))
|
||||
for i, e := range merged {
|
||||
texts[i] = e.Text
|
||||
@@ -107,7 +131,7 @@ func main() {
|
||||
h := sha256.Sum256([]byte(strings.Join(texts, "\n")))
|
||||
datasetHash := hex.EncodeToString(h[:16])
|
||||
|
||||
// 7. Stats
|
||||
// 8. Stats
|
||||
routeCounts := make(map[semantic.SemanticRoute]int)
|
||||
fpCount, resCount := 0, 0
|
||||
for _, e := range merged {
|
||||
@@ -124,7 +148,7 @@ func main() {
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "fast-path: %d residual: %d\n", fpCount, resCount)
|
||||
|
||||
// 8. Write merged corpus
|
||||
// 9. Write merged corpus
|
||||
outEnv := semantic.CorpusEnvelope{
|
||||
SchemaVersion: 1,
|
||||
Name: "semantic_coarse_route_v1",
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"os"
|
||||
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/router/semantic"
|
||||
)
|
||||
|
||||
// headsMain runs the deployed cascade minus the resident LLM: stage-0
|
||||
@@ -48,7 +49,7 @@ func headsMain(poolPath, outPath string) {
|
||||
}
|
||||
defer heads.Close()
|
||||
|
||||
acts := router.DefaultActMatcher{Fns: actVerbList()}
|
||||
acts := router.DefaultActMatcher{Fns: semantic.ExperimentActVerbs()}
|
||||
r := router.New(router.Config{
|
||||
Grammars: router.StageZeroGrammars(acts),
|
||||
Classifier: cls,
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/router/semantic"
|
||||
)
|
||||
|
||||
// Seed loading replicated from cmd/mavend/voicewire.go (seedClassifier) and
|
||||
@@ -23,26 +24,13 @@ var seedIntents = []router.Intent{
|
||||
router.IntentNote, router.IntentQuery, router.IntentChat, router.IntentSystem,
|
||||
}
|
||||
|
||||
// actVerbList mirrors the eval fixture's static allowlist
|
||||
// (internal/router/semantic/helpers_test.go). The production matcher's
|
||||
// allowlist is the deployment's enabled tools; an act grammar can only catch
|
||||
// a row whose first tokens match an allowlisted verb, and the experiment's
|
||||
// act allowlist is the one the accepted slice-21 methodology used.
|
||||
func actVerbList() []string {
|
||||
return []string{
|
||||
"перезапусти", "перезагрузи", "выключи", "включи", "останови", "запусти",
|
||||
"закрой", "открой", "сделай", "поставь",
|
||||
"restart", "reboot", "stop", "start", "turn off", "turn on", "open", "close",
|
||||
}
|
||||
}
|
||||
|
||||
// buildMinimalRouter reproduces internal/router/semantic/buildMinimalRouter:
|
||||
// the daemon's grammar set, a hash-embedder classifier seeded from
|
||||
// models/seeds, and the deployed 0.55 threshold. Deterministic and
|
||||
// reproducible. The ONNX embedder and the routing heads score elsewhere;
|
||||
// this is the floor the eval fixture reports as the legacy baseline.
|
||||
func buildMinimalRouter() *router.Router {
|
||||
acts := router.DefaultActMatcher{Fns: actVerbList()}
|
||||
acts := router.DefaultActMatcher{Fns: semantic.ExperimentActVerbs()}
|
||||
cls := router.NewClassifier(router.NewHashEmbedder(1024))
|
||||
seedClassifier(cls)
|
||||
return router.New(router.Config{
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
// slice23 — fast-path metadata reconciliation diagnostic.
|
||||
//
|
||||
// Classifies every disagreement between the corpus's stored fast_path_resolved
|
||||
// flag and what the production fast path derives today (TryFastPath over the
|
||||
// stage-0 grammars with the experiment's act allowlist). Outputs a JSON
|
||||
// decomposition and a console summary for docs/evals reports.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// go run ./cmd/semantic-router-experiment/slice23/ -out /tmp/mvn-s23/drift.json
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
|
||||
"github.com/kami/maven/internal/router"
|
||||
"github.com/kami/maven/internal/router/semantic"
|
||||
)
|
||||
|
||||
type conflict struct {
|
||||
Text string `json:"text"`
|
||||
SourceID string `json:"source_id"`
|
||||
Route string `json:"route"`
|
||||
Source string `json:"source"`
|
||||
Group string `json:"split_group"`
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
Dev bool `json:"dev"`
|
||||
// Direction: claimed_fast_now_miss = stored fast, runtime residual;
|
||||
// mirror_missed = stored residual, runtime fast.
|
||||
Direction string `json:"direction"`
|
||||
Grammar string `json:"grammar,omitempty"`
|
||||
// ShapeDeclined is true when at least one stage-0 grammar matched the
|
||||
// utterance's shape but refused the content (falls through like the router).
|
||||
ShapeDeclined bool `json:"shape_declined"`
|
||||
// DeclinedGrammars names every stage-0 grammar that matched the shape but
|
||||
// declined the content, for claimed_fast_now_miss rows.
|
||||
DeclinedGrammars []string `json:"declined_grammars,omitempty"`
|
||||
}
|
||||
|
||||
type report struct {
|
||||
Meta metaSummary `json:"meta"`
|
||||
Pop popSummary `json:"population"`
|
||||
Conflicts []conflict `json:"conflicts"`
|
||||
ByRoute map[string]map[string]int `json:"by_route"`
|
||||
ByGrammar map[string]int `json:"by_grammar"`
|
||||
BySource map[string]map[string]int `json:"by_source"`
|
||||
ByFamily map[string]map[string]int `json:"by_family"`
|
||||
Direction map[string]int `json:"by_direction"`
|
||||
Declined int `json:"claimed_fast_with_declined_shape"`
|
||||
NoShape int `json:"claimed_fast_with_no_shape"`
|
||||
}
|
||||
|
||||
type metaSummary struct {
|
||||
Total int `json:"total"`
|
||||
DevCount int `json:"dev_count"`
|
||||
Frozen int `json:"frozen_count"`
|
||||
}
|
||||
|
||||
type popSummary struct {
|
||||
StoredFast int `json:"stored_fast"`
|
||||
StoredResid int `json:"stored_residual"`
|
||||
DerivedFast int `json:"derived_fast"`
|
||||
DerivedResid int `json:"derived_residual"`
|
||||
// Dev-pool residual route counts derived as the router sees them today.
|
||||
DevResidualByRoute map[string]int `json:"dev_residual_by_route"`
|
||||
// Dev-pool residual non-action + action OOD as derived.
|
||||
DevResidualNonAction int `json:"dev_residual_non_action"`
|
||||
DevResidualAction int `json:"dev_residual_action"`
|
||||
// Fast rows in the dev pool, derived.
|
||||
DevFast int `json:"dev_fast"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
outPath := flag.String("out", "/tmp/mvn-s23/drift.json", "output JSON path")
|
||||
flag.Parse()
|
||||
|
||||
exs, err := semantic.LoadCorpus()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "load corpus: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
_, dev, _ := semantic.FrozenHoldoutSplit(exs)
|
||||
devSet := make(map[string]bool, len(dev))
|
||||
for _, e := range dev {
|
||||
devSet[e.SourceID] = true
|
||||
}
|
||||
|
||||
// Stage-zero grammar list for shape/declined attribution (same list the
|
||||
// derivation walks).
|
||||
acts := router.DefaultActMatcher{Fns: semantic.ExperimentActVerbs()}
|
||||
gs := router.StageZeroGrammars(acts)
|
||||
|
||||
var (
|
||||
conflicts []conflict
|
||||
byRoute = map[string]map[string]int{}
|
||||
byGrammar = map[string]int{}
|
||||
bySource = map[string]map[string]int{}
|
||||
byFamily = map[string]map[string]int{}
|
||||
byDirection = map[string]int{}
|
||||
storedFast, derivedFast, declined, noShape int
|
||||
devResidByRoute = map[string]int{}
|
||||
devResidNonAct, devResidAct, devFast int
|
||||
)
|
||||
|
||||
for _, e := range exs {
|
||||
o := semantic.DeriveFastPath(e.Text)
|
||||
inDev := devSet[e.SourceID]
|
||||
|
||||
st := e.FastPathResolved
|
||||
if st {
|
||||
storedFast++
|
||||
}
|
||||
if o.Matched {
|
||||
derivedFast++
|
||||
}
|
||||
|
||||
var c *conflict
|
||||
switch {
|
||||
case st && o.Matched:
|
||||
case st && !o.Matched:
|
||||
// Stored fast but the runtime misses. Attribute why.
|
||||
shapeDeclined := false
|
||||
var declinedNames []string
|
||||
for _, g := range gs {
|
||||
_, matched, ok := g.Evaluate(e.Text)
|
||||
if matched && !ok {
|
||||
shapeDeclined = true
|
||||
declinedNames = append(declinedNames, g.Name)
|
||||
}
|
||||
}
|
||||
if shapeDeclined {
|
||||
declined++
|
||||
} else {
|
||||
noShape++
|
||||
}
|
||||
c = &conflict{Direction: "claimed_fast_now_miss", ShapeDeclined: shapeDeclined, DeclinedGrammars: declinedNames}
|
||||
case !st && o.Matched:
|
||||
c = &conflict{Direction: "mirror_missed", Grammar: o.Grammar}
|
||||
}
|
||||
|
||||
if c != nil {
|
||||
c.Text = e.Text
|
||||
c.SourceID = e.SourceID
|
||||
c.Route = string(e.Route)
|
||||
c.Source = e.Source
|
||||
c.Group = e.SplitGroup
|
||||
c.Tags = e.Tags
|
||||
c.Dev = inDev
|
||||
conflicts = append(conflicts, *c)
|
||||
byDirection[c.Direction]++
|
||||
byGrammar[c.Grammar]++
|
||||
if byRoute[c.Direction] == nil {
|
||||
byRoute[c.Direction] = map[string]int{}
|
||||
}
|
||||
byRoute[c.Direction][c.Route]++
|
||||
if bySource[c.Direction] == nil {
|
||||
bySource[c.Direction] = map[string]int{}
|
||||
}
|
||||
bySource[c.Direction][c.Source]++
|
||||
if byFamily[c.Direction] == nil {
|
||||
byFamily[c.Direction] = map[string]int{}
|
||||
}
|
||||
byFamily[c.Direction][c.Group]++
|
||||
}
|
||||
|
||||
if inDev {
|
||||
if o.Matched {
|
||||
devFast++
|
||||
} else {
|
||||
devResidByRoute[string(e.Route)]++
|
||||
if e.Route == semantic.RouteAction {
|
||||
devResidAct++
|
||||
} else {
|
||||
devResidNonAct++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(conflicts, func(i, j int) bool { return conflicts[i].SourceID < conflicts[j].SourceID })
|
||||
|
||||
rep := report{
|
||||
Meta: metaSummary{Total: len(exs), DevCount: len(dev), Frozen: len(exs) - len(dev)},
|
||||
Pop: popSummary{
|
||||
StoredFast: storedFast, StoredResid: len(exs) - storedFast,
|
||||
DerivedFast: derivedFast, DerivedResid: len(exs) - derivedFast,
|
||||
DevResidualByRoute: devResidByRoute,
|
||||
DevResidualNonAction: devResidNonAct, DevResidualAction: devResidAct,
|
||||
DevFast: devFast,
|
||||
},
|
||||
Conflicts: conflicts,
|
||||
ByRoute: byRoute, ByGrammar: byGrammar, BySource: bySource, ByFamily: byFamily,
|
||||
Direction: byDirection, Declined: declined, NoShape: noShape,
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(rep, "", " ")
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "marshal: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := os.WriteFile(*outPath, data, 0644); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "write %s: %v\n", *outPath, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fmt.Printf("total %d (dev %d, frozen %d)\n", rep.Meta.Total, rep.Meta.DevCount, rep.Meta.Frozen)
|
||||
fmt.Printf("stored fast=%d residual=%d\n", rep.Pop.StoredFast, rep.Pop.StoredResid)
|
||||
fmt.Printf("derived fast=%d residual=%d\n", rep.Pop.DerivedFast, rep.Pop.DerivedResid)
|
||||
fmt.Printf("disagreements total %d\n", len(conflicts))
|
||||
for _, d := range []string{"claimed_fast_now_miss", "mirror_missed"} {
|
||||
fmt.Printf(" %-22s %d\n", d, byDirection[d])
|
||||
if d == "claimed_fast_now_miss" {
|
||||
fmt.Printf(" with declined shape: %d no shape: %d\n", declined, noShape)
|
||||
}
|
||||
}
|
||||
fmt.Println(" mirror-missed by grammar:")
|
||||
for _, k := range sortedKeys(byGrammar) {
|
||||
fmt.Printf(" %-28s %d\n", k, byGrammar[k])
|
||||
}
|
||||
fmt.Println(" by route:")
|
||||
for _, d := range sortedKeys(byRoute) {
|
||||
fmt.Printf(" %-22s %v\n", d, byRoute[d])
|
||||
}
|
||||
fmt.Printf("dev pool derived: fast=%d residual=%d (non-action=%d action=%d)\n",
|
||||
rep.Pop.DevFast, rep.Pop.DevResidualNonAction+rep.Pop.DevResidualAction,
|
||||
rep.Pop.DevResidualNonAction, rep.Pop.DevResidualAction)
|
||||
fmt.Printf("dev residual by route: %v\n", rep.Pop.DevResidualByRoute)
|
||||
fmt.Printf("wrote %s\n", *outPath)
|
||||
}
|
||||
|
||||
func sortedKeys[T any](m map[string]T) []string {
|
||||
ks := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
ks = append(ks, k)
|
||||
}
|
||||
sort.Strings(ks)
|
||||
return ks
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Slice 23 emit: five-way residual non-action semantic router — data files
|
||||
========================================================================
|
||||
|
||||
Slice 23 reconciles corpus fast-path metadata with the production router
|
||||
(TryFastPath over stage-0 grammars). The corpus builder no longer mirrors the
|
||||
grammars by hand; fast_path_resolved is derived from the router, so this emit
|
||||
flags exactly the rows the router genuinely leaves for the general cascade.
|
||||
|
||||
This script only repackages the frozen dev pool for the Go legacy baseline and
|
||||
the Python experiment, writing into /tmp/mvn-s23 so the slice-22 artifacts
|
||||
stay untouched. Logic is slice22_emit.py verbatim; only OUT_DIR differs.
|
||||
|
||||
/tmp/mvn-s23/pool.json residual non-action dev rows: idx, text, n_text,
|
||||
route, tags, cv_fold, split_group, family_id,
|
||||
source_id (1509 rows)
|
||||
/tmp/mvn-s23/ood.json residual ACTION dev rows (720): same shape; OOD
|
||||
probes only, never primary metrics
|
||||
/tmp/mvn-s23/stats.json population summary (routes, families, folds)
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, HERE)
|
||||
|
||||
import slice18_sparse # noqa: E402 (normalize_match_text, load_data, filters)
|
||||
|
||||
OUT_DIR = "/tmp/mvn-s23"
|
||||
|
||||
ROUTES = ["conversation", "knowledge", "memory_write", "system", "uncertain"]
|
||||
|
||||
|
||||
def main():
|
||||
meta, examples = slice18_sparse.load_data()
|
||||
dev = slice18_sparse.filter_dev_pool(examples)
|
||||
print(f"dev pool: {len(dev)} rows "
|
||||
f"(meta declares dev_count={meta.get('dev_count')})")
|
||||
|
||||
rows = []
|
||||
for i, e in enumerate(dev):
|
||||
if not e["fast_path_resolved"]:
|
||||
rows.append({
|
||||
"idx": i,
|
||||
"text": e["text"],
|
||||
"n_text": slice18_sparse.normalize_match_text(e["text"]),
|
||||
"route": e["route"],
|
||||
"tags": sorted(set(e.get("tags", []))),
|
||||
"cv_fold": e["cv_fold"],
|
||||
"split_group": e["split_group"],
|
||||
"family_id": e["family_id"],
|
||||
"source_id": e["source_id"],
|
||||
})
|
||||
|
||||
na = [r for r in rows if r["route"] != "action"]
|
||||
ood = [r for r in rows if r["route"] == "action"]
|
||||
print(f"residual rows: {len(rows)} non-action: {len(na)} action(OOD): {len(ood)}")
|
||||
|
||||
by_route = {}
|
||||
for r in na:
|
||||
by_route[r["route"]] = by_route.get(r["route"], 0) + 1
|
||||
print("routes:", by_route)
|
||||
assert sum(by_route.values()) == len(na)
|
||||
assert set(ROUTES) == set(by_route), "route set must be the five-way"
|
||||
|
||||
by_family = {}
|
||||
for r in na:
|
||||
by_family[r["family_id"]] = by_family.get(r["family_id"], 0) + 1
|
||||
by_fold = {}
|
||||
for r in na:
|
||||
by_fold[r["cv_fold"]] = by_fold.get(r["cv_fold"], 0) + 1
|
||||
print(f"family_ids: {len(by_family)} split_groups: {len(set(r['split_group'] for r in na))}")
|
||||
print("folds:", by_fold)
|
||||
|
||||
os.makedirs(OUT_DIR, exist_ok=True)
|
||||
with open(os.path.join(OUT_DIR, "pool.json"), "w") as f:
|
||||
json.dump(na, f, ensure_ascii=False, indent=1)
|
||||
with open(os.path.join(OUT_DIR, "ood.json"), "w") as f:
|
||||
json.dump(ood, f, ensure_ascii=False, indent=1)
|
||||
with open(os.path.join(OUT_DIR, "stats.json"), "w") as f:
|
||||
json.dump({
|
||||
"dev_count": len(dev),
|
||||
"residual_count": len(rows),
|
||||
"non_action_count": len(na),
|
||||
"action_ood_count": len(ood),
|
||||
"routes": by_route,
|
||||
"family_ids": len(by_family),
|
||||
"folds": by_fold,
|
||||
"top_family": dict(sorted(by_family.items(), key=lambda kv: -kv[1])[:15]),
|
||||
}, f, ensure_ascii=False, indent=1)
|
||||
print(f"wrote {OUT_DIR}/{{pool,ood,stats}}.json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,559 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Slice 23: five-way residual non-action semantic router — corrected population
|
||||
============================================================================
|
||||
|
||||
Slice 22 reported the corpus's fast-path mirror was stale next to the
|
||||
production stage-0 grammars (185 residual rows resolved at runtime). Slice 23
|
||||
derives fast_path_resolved from the real router, rebuilds the corpus, and
|
||||
re-measures the primary slice-22 results on the corrected residual pool.
|
||||
Logic and configs are slice22_main.py verbatim; only OUT_DIR differs.
|
||||
|
||||
Population: the corrected dev-pool residual non-action rows (1509; the pool
|
||||
written by slice23_emit.py). Corrected action rows (720) are OOD probes only.
|
||||
|
||||
Metrics written to /tmp/mvn-s23/results.json:
|
||||
§1 population
|
||||
§2 legacy baseline (legacy.json / legacy_heads.json): acc, macro-F1,
|
||||
per-class P/R/F1, confusion, illegal_action_prediction count
|
||||
§3 e5-linear primary head: C grid, grouped CV OOF, per-fold P/R/F1 +
|
||||
variance + composition
|
||||
§4 floors: majority, centroid (cosine nearest-mean), sparse word+char
|
||||
TF-IDF logistic (slice18 builder), all grouped CV
|
||||
§5 route-family (family_id) leave-family-out
|
||||
§6 knowledge vs memory_write: matched pairs (water/homelab/task) ordering
|
||||
§7 uncertain as an explicit class: P/R/F1 + top confusions
|
||||
§8 OOF confidence: max-softmax correct/wrong, ECE, log-loss, Brier,
|
||||
coverage/accuracy/macro-F1 abstention curves (no threshold chosen)
|
||||
§9 action OOD probes: fold models applied to the corrected action rows
|
||||
§10 artifact cost: head params, serialized bytes, incremental head latency
|
||||
|
||||
No corpus label is changed. No frozen-holdout rows are inspected.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, HERE)
|
||||
|
||||
import slice18_sparse # noqa: E402
|
||||
import slice19_main # noqa: E402
|
||||
|
||||
EMB_PATH = "/tmp/mvn-experiment/embeddings.json"
|
||||
OUT_DIR = "/tmp/mvn-s23"
|
||||
CLASSES = ["conversation", "knowledge", "memory_write", "system", "uncertain"]
|
||||
CLASS_PREFIX = ["conversation", "knowledge", "memory_write", "system", "uncertain"]
|
||||
C_GRID = [0.1, 1.0, 10.0]
|
||||
|
||||
# Route-family holdouts the report calls out by name (slice-22 brief): every
|
||||
# family that is not part of the shared subject inventory on either side.
|
||||
HOLDOUT_GROUPS = {
|
||||
"capability": ["knowledge:capability-ha", "knowledge:capability-tool"],
|
||||
"world": ["knowledge:world-def", "knowledge:world-explain"],
|
||||
"calendar": ["knowledge:calendar", "knowledge:calendar-time", "knowledge:calendar-next"],
|
||||
"recall": ["knowledge:recall-fact", "knowledge:recall-note", "knowledge:recall-possessive"],
|
||||
"fact": ["fact:meal", "fact:water", "fact:sleep", "fact:shower", "fact:break", "fact:pills", "fact:exercise"],
|
||||
"note": ["note:idea", "note:homelab", "note:task"],
|
||||
"remember": ["free:remember"],
|
||||
"system": None, # all system:*
|
||||
"conversation": None,
|
||||
"uncertain": None,
|
||||
}
|
||||
|
||||
|
||||
def load_pool_and_embeds():
|
||||
with open(os.path.join(OUT_DIR, "pool.json")) as f:
|
||||
pool = json.load(f)
|
||||
meta, examples = slice18_sparse.load_data()
|
||||
dev = slice18_sparse.filter_dev_pool(examples)
|
||||
by_idx = {e["dev_idx"]: e for e in dev} if "dev_idx" in dev[0] else None
|
||||
# pool rows carry idx = position among dev_pool rows in dev order
|
||||
emb_by_idx = {i: np.asarray(e["embedding"], dtype=np.float64)
|
||||
for i, e in enumerate(dev)}
|
||||
for r in pool:
|
||||
r["emb"] = emb_by_idx[r["idx"]]
|
||||
r["y"] = r["route"]
|
||||
return pool, meta
|
||||
|
||||
|
||||
def oof_proba_grouped(X, y, folds, C=1.0):
|
||||
"""Grouped OOF probability matrix (n×5, class order CLASSES)."""
|
||||
y_idx = np.array([CLASSES.index(c) for c in y])
|
||||
folds = np.asarray(folds)
|
||||
proba = np.zeros((len(y_idx), len(CLASSES)))
|
||||
for te_fold in sorted(set(folds.tolist())):
|
||||
tr = folds != te_fold
|
||||
te = folds == te_fold
|
||||
clf = slice18_sparse.LogisticRegression(
|
||||
C=C, max_iter=2000, solver="lbfgs", random_state=42)
|
||||
clf.fit(X[tr], y_idx[tr])
|
||||
proba[te] = clf.predict_proba(X[te])
|
||||
return proba
|
||||
|
||||
|
||||
def cls_metrics(yt, yp):
|
||||
import sklearn.metrics as m
|
||||
yt = np.asarray(yt)
|
||||
yp = np.asarray(yp)
|
||||
if yt.dtype != np.int64 and yt.dtype != np.int32:
|
||||
yt = np.array([CLASSES.index(c) for c in yt])
|
||||
if yp.dtype != np.int64 and yp.dtype != np.int32:
|
||||
yp = np.array([CLASSES.index(c) for c in yp])
|
||||
labels = list(range(len(CLASSES)))
|
||||
n = len(yt)
|
||||
acc = m.accuracy_score(yt, yp)
|
||||
macro = m.f1_score(yt, yp, average="macro", labels=labels, zero_division=0)
|
||||
pr, rc, f1, sup = m.precision_recall_fscore_support(
|
||||
yt, yp, labels=labels, zero_division=0)
|
||||
per = {c: {"p": float(pr[i]), "r": float(rc[i]), "f1": float(f1[i]), "n": int(sup[i])}
|
||||
for i, c in enumerate(CLASSES)}
|
||||
conf = m.confusion_matrix(yt, yp, labels=labels).tolist()
|
||||
return {"n": n, "acc": acc, "macro_f1": macro, "per_class": per, "confusion": conf}
|
||||
|
||||
|
||||
def fold_report(yt, proba, folds, true_y):
|
||||
out = {}
|
||||
folds_arr = np.asarray(folds)
|
||||
comp = {}
|
||||
for f in sorted(set(folds_arr.tolist())):
|
||||
mask = folds_arr == f
|
||||
yt_f = [CLASSES.index(y) for y in true_y[mask]]
|
||||
comp[f] = {c: int((np.array(true_y[mask]) == c).sum()) for c in CLASSES}
|
||||
per_fold = {}
|
||||
for f in sorted(set(folds_arr.tolist())):
|
||||
mask = folds_arr == f
|
||||
yp = proba[mask].argmax(1).tolist()
|
||||
m = cls_metrics([yt[i] for i in np.where(mask)[0].tolist()], yp)
|
||||
per_fold[f] = {"acc": m["acc"], "macro_f1": m["macro_f1"]}
|
||||
out["composition"] = comp
|
||||
out["per_fold"] = per_fold
|
||||
accs = [v["acc"] for v in per_fold.values()]
|
||||
macros = [v["macro_f1"] for v in per_fold.values()]
|
||||
out["acc_mean"] = float(np.mean(accs))
|
||||
out["acc_std"] = float(np.std(accs))
|
||||
out["macro_f1_mean"] = float(np.mean(macros))
|
||||
out["macro_f1_std"] = float(np.std(macros))
|
||||
return out
|
||||
|
||||
|
||||
def ece(yt, proba, n_bins=15):
|
||||
conf = proba.max(1)
|
||||
pred = proba.argmax(1)
|
||||
acc = (pred == yt).astype(float)
|
||||
bins = np.linspace(0, 1, n_bins + 1)
|
||||
tot = 0.0
|
||||
details = []
|
||||
counts = 0
|
||||
for i in range(n_bins):
|
||||
lo, hi = bins[i], bins[i + 1]
|
||||
m = (conf >= lo) & (conf < hi) if i < n_bins - 1 else conf >= lo
|
||||
if m.sum() == 0:
|
||||
continue
|
||||
acc_m = acc[m].mean()
|
||||
conf_m = conf[m].mean()
|
||||
w = m.sum() / len(conf)
|
||||
tot += w * abs(acc_m - conf_m)
|
||||
counts += int(m.sum())
|
||||
details.append({"bin": i, "lo": lo, "hi": hi, "conf": float(conf_m),
|
||||
"acc": float(acc_m), "n": int(m.sum())})
|
||||
return {"ece": float(tot), "n_bins": n_bins, "counted": counts, "bins": details}
|
||||
|
||||
|
||||
def main():
|
||||
pool, meta = load_pool_and_embeds()
|
||||
pool.sort(key=lambda r: r["idx"])
|
||||
print(f"pool: {len(pool)} rows")
|
||||
|
||||
from sklearn.metrics import brier_score_loss, log_loss
|
||||
|
||||
report = {"population": {}, "legacy": {}, "e5_linear": {}, "floors": {},
|
||||
"family_holdouts": {}, "kmw": {}, "uncertain": {}, "confidence": {},
|
||||
"ood": {}, "artifact": {}}
|
||||
|
||||
# ── §1 population ──────────────────────────────────────────────────────
|
||||
cnt = {}
|
||||
for r in pool:
|
||||
cnt[r["y"]] = cnt.get(r["y"], 0) + 1
|
||||
report["population"] = {
|
||||
"n": len(pool),
|
||||
"routes": cnt,
|
||||
"family_ids": len(set(r["family_id"] for r in pool)),
|
||||
"split_groups": len(set(r["split_group"] for r in pool)),
|
||||
"folds": {str(f): int(sum(1 for r in pool if r["cv_fold"] == f)) for f in sorted(set(r["cv_fold"] for r in pool))},
|
||||
"corpus": {k: v for k, v in meta.items() if k in
|
||||
("dev_count", "residual_count", "fast_path_count",
|
||||
"dimension", "embedder_id", "input_template", "pooling", "normalization")},
|
||||
}
|
||||
print("\n§1 population:", report["population"])
|
||||
|
||||
X = np.vstack([r["emb"] for r in pool])
|
||||
y = np.array([r["y"] for r in pool])
|
||||
folds = np.array([r["cv_fold"] for r in pool])
|
||||
yt = np.array([CLASSES.index(c) for c in y])
|
||||
|
||||
# ── §2 legacy baselines ────────────────────────────────────────────────
|
||||
import collections
|
||||
for tag, fname in [("hash", "legacy.json"), ("heads", "legacy_heads.json")]:
|
||||
path = os.path.join(OUT_DIR, fname)
|
||||
if not os.path.exists(path):
|
||||
continue
|
||||
leg = json.load(open(path))
|
||||
leg_by_idx = {r["idx"]: r for r in leg}
|
||||
yp_leg = []
|
||||
illegal = []
|
||||
for r in pool:
|
||||
lr = leg_by_idx[r["idx"]]
|
||||
if lr["illegal_action_prediction"]:
|
||||
illegal.append(lr)
|
||||
yp_leg.append("action")
|
||||
else:
|
||||
yp_leg.append(lr["class"])
|
||||
yp_leg = np.array(yp_leg)
|
||||
# five-way: an 'action' prediction is an error (outside the label set)
|
||||
yp5 = np.array([("uncertain" if p == "action" else p) for p in yp_leg])
|
||||
m = cls_metrics(y, yp5)
|
||||
m["illegal_action_prediction"] = len(illegal)
|
||||
m["illegal_cases"] = [{"idx": i["idx"], "text": i["text"], "route": i["route"],
|
||||
"intent": i["intent"], "producer": i["producer"],
|
||||
"confidence": i["confidence"]} for i in illegal]
|
||||
# per-cell confusion also shows 'action' column
|
||||
conf_counts = collections.Counter(zip(y, yp_leg))
|
||||
m["confusion_with_action"] = {f"{a}->{b}": int(c) for (a, b), c in conf_counts.items()}
|
||||
report["legacy"][tag] = m
|
||||
print(f"\n§2 legacy ({tag}) acc={m['acc']:.4f} macroF1={m['macro_f1']:.4f} "
|
||||
f"illegal={len(illegal)}")
|
||||
for c in CLASSES:
|
||||
p = m["per_class"][c]
|
||||
print(f" {c:<14} P={p['p']:.3f} R={p['r']:.3f} F1={p['f1']:.3f} n={p['n']}")
|
||||
|
||||
# grammar-pure residual: rows not resolved by any current stage-0 grammar
|
||||
if os.path.exists(os.path.join(OUT_DIR, "legacy.json")):
|
||||
leg = json.load(open(os.path.join(OUT_DIR, "legacy.json")))
|
||||
gh = {r["idx"] for r in leg if r["producer"] == "grammar"}
|
||||
gp_mask = np.array([r["idx"] not in gh for r in pool])
|
||||
report["grammar_drift"] = {
|
||||
"grammar_hits_in_pool": len(gh),
|
||||
"grammar_pure_n": int(gp_mask.sum()),
|
||||
}
|
||||
|
||||
# ── §3 e5-linear primary head ─────────────────────────────────────────
|
||||
print("\n§3 e5-linear")
|
||||
bestC, bestMac = 1.0, -1.0
|
||||
grid = {}
|
||||
for C in C_GRID:
|
||||
p = oof_proba_grouped(X, y, folds, C=C)
|
||||
mp = cls_metrics(y, p.argmax(1).tolist())
|
||||
grid[float(C)] = {"acc": mp["acc"], "macro_f1": mp["macro_f1"]}
|
||||
print(f" C={C} acc={mp['acc']:.4f} macroF1={mp['macro_f1']:.4f}")
|
||||
if mp["macro_f1"] > bestMac:
|
||||
bestMac, bestC = mp["macro_f1"], C
|
||||
print(f" -> best C={bestC}")
|
||||
p_best = oof_proba_grouped(X, y, folds, C=bestC)
|
||||
m_best = cls_metrics(y, p_best.argmax(1).tolist())
|
||||
m_best["C"] = bestC
|
||||
m_best["C_grid"] = grid
|
||||
m_best["folds"] = fold_report(yt, p_best, folds, y)
|
||||
report["e5_linear"] = m_best
|
||||
for f, v in m_best["folds"]["per_fold"].items():
|
||||
print(f" fold {f}: acc={v['acc']:.4f} macroF1={v['macro_f1']:.4f}")
|
||||
print(f" fold acc mean={m_best['folds']['acc_mean']:.4f} "
|
||||
f"std={m_best['folds']['acc_std']:.4f}; "
|
||||
f"macroF1 mean={m_best['folds']['macro_f1_mean']:.4f} "
|
||||
f"std={m_best['folds']['macro_f1_std']:.4f}")
|
||||
for c in CLASSES:
|
||||
p_ = m_best["per_class"][c]
|
||||
print(f" {c:<14} P={p_['p']:.3f} R={p_['r']:.3f} F1={p_['f1']:.3f} n={p_['n']}")
|
||||
|
||||
# grammar-pure sensitivity for the primary head
|
||||
if "grammar_drift" in report:
|
||||
mp_gp = cls_metrics(y[gp_mask], p_best[gp_mask].argmax(1).tolist())
|
||||
report["e5_linear"]["grammar_pure"] = {
|
||||
"acc": mp_gp["acc"], "macro_f1": mp_gp["macro_f1"], "n": int(gp_mask.sum())}
|
||||
|
||||
# ── §4 floors ─────────────────────────────────────────────────────────
|
||||
print("\n§4 floors")
|
||||
# majority floor
|
||||
maj = CLASSES.index("knowledge")
|
||||
ym = np.full(len(y), maj)
|
||||
mm = cls_metrics(y, ym)
|
||||
report["floors"]["majority"] = {"acc": mm["acc"], "macro_f1": mm["macro_f1"],
|
||||
"per_class": mm["per_class"]}
|
||||
print(f" majority (predict {CLASSES[maj]}): acc={mm['acc']:.4f} macroF1={mm['macro_f1']:.4f}")
|
||||
|
||||
# centroid floor: cosine to per-class mean of the training folds' embeddings
|
||||
cf_proba = np.zeros((len(yt), len(CLASSES)))
|
||||
folds_arr = np.asarray(folds)
|
||||
for te_fold in sorted(set(folds_arr.tolist())):
|
||||
tr = folds_arr != te_fold
|
||||
te = folds_arr == te_fold
|
||||
centroids = []
|
||||
for c in CLASSES:
|
||||
idxs = np.where(tr & (y == c))[0]
|
||||
ctr = X[idxs].mean(axis=0)
|
||||
ctr = ctr / np.linalg.norm(ctr)
|
||||
centroids.append(ctr)
|
||||
Cm = np.vstack(centroids)
|
||||
sims = X[te] @ Cm.T
|
||||
cf_proba[te] = sims
|
||||
yc = cf_proba.argmax(1)
|
||||
# accuracy + macroF1 with the same 5-way
|
||||
mc = cls_metrics(y, yc.tolist())
|
||||
report["floors"]["centroid"] = {"acc": mc["acc"], "macro_f1": mc["macro_f1"],
|
||||
"per_class": mc["per_class"]}
|
||||
print(f" centroid cosine: acc={mc['acc']:.4f} macroF1={mc['macro_f1']:.4f}")
|
||||
|
||||
# sparse word+char logistic (slice18 builder, grouped CV, five-way)
|
||||
texts = [r["n_text"] for r in pool]
|
||||
Xs, _vec = slice18_sparse.build_features(texts, "both")
|
||||
psp = np.zeros((len(yt), len(CLASSES)))
|
||||
for te_fold in sorted(set(folds_arr.tolist())):
|
||||
tr = folds_arr != te_fold
|
||||
te = folds_arr == te_fold
|
||||
clf = slice18_sparse.LogisticRegression(
|
||||
C=1.0, max_iter=2000, solver="lbfgs", random_state=42)
|
||||
clf.fit(Xs[tr], yt[tr])
|
||||
psp[te] = clf.predict_proba(Xs[te])
|
||||
msp = cls_metrics(y, psp.argmax(1).tolist())
|
||||
report["floors"]["sparse_word_char"] = {
|
||||
"acc": msp["acc"], "macro_f1": msp["macro_f1"], "per_class": msp["per_class"],
|
||||
"vocab": slice18_sparse.vocab_size(_vec)}
|
||||
print(f" sparse both: acc={msp['acc']:.4f} macroF1={msp['macro_f1']:.4f} "
|
||||
f"vocab={report['floors']['sparse_word_char']['vocab']}")
|
||||
|
||||
# ── §5 route-family holdouts ─────────────────────────────────────────
|
||||
print("\n§5 route-family holdouts")
|
||||
fam = np.array([r["family_id"] for r in pool])
|
||||
holdouts = {}
|
||||
all_fams = sorted(set(fam.tolist()))
|
||||
for grp, fams in HOLDOUT_GROUPS.items():
|
||||
if fams is None:
|
||||
fams = [f for f in all_fams if f.startswith(grp + ":")]
|
||||
mask = np.isin(fam, fams)
|
||||
if mask.sum() == 0:
|
||||
continue
|
||||
tr = ~mask
|
||||
clf = slice18_sparse.LogisticRegression(
|
||||
C=bestC, max_iter=2000, solver="lbfgs", random_state=42)
|
||||
clf.fit(X[tr], yt[tr])
|
||||
ypgrp = clf.predict(X[mask])
|
||||
m = cls_metrics([CLASSES.index(c) for c in y[mask]], ypgrp.tolist())
|
||||
m["families"] = fams
|
||||
m["rows"] = int(mask.sum())
|
||||
holdouts[grp] = {"acc": m["acc"], "macro_f1": m["macro_f1"], "n": int(mask.sum()),
|
||||
"per_class": m["per_class"]}
|
||||
print(f" {grp:<14} n={m['rows']} acc={m['acc']:.4f} macroF1={m['macro_f1']:.4f}")
|
||||
# full leave-one-family-out summary
|
||||
lofo_accs = []
|
||||
lofo_f1s = []
|
||||
for f in all_fams:
|
||||
mask = fam == f
|
||||
tr = ~mask
|
||||
clf = slice18_sparse.LogisticRegression(
|
||||
C=bestC, max_iter=2000, solver="lbfgs", random_state=42)
|
||||
clf.fit(X[tr], yt[tr])
|
||||
ypf = clf.predict(X[mask])
|
||||
m = cls_metrics([CLASSES.index(c) for c in y[mask]], ypf.tolist())
|
||||
lofo_accs.append(m["acc"])
|
||||
lofo_f1s.append(m["macro_f1"])
|
||||
holdouts["_all_49_lo_"] = {"n_families": len(all_fams),
|
||||
"acc_mean": float(np.mean(lofo_accs)),
|
||||
"macro_f1_mean": float(np.mean(lofo_f1s))}
|
||||
report["family_holdouts"] = holdouts
|
||||
print(f" leave-one-family-out over {len(all_fams)} families: "
|
||||
f"acc mean={np.mean(lofo_accs):.4f} macroF1 mean={np.mean(lofo_f1s):.4f}")
|
||||
|
||||
# ── §6 knowledge vs memory_write ─────────────────────────────────────
|
||||
print("\n§6 knowledge vs memory_write")
|
||||
# reuse e5-linear OOF: does the model put the higher probability on the
|
||||
# right side (memory_write for a write, knowledge for a recall)?
|
||||
conf_km = np.zeros((2, 2))
|
||||
pk = p_best[:, CLASSES.index("knowledge")]
|
||||
pmw = p_best[:, CLASSES.index("memory_write")]
|
||||
for i in range(len(yt)):
|
||||
t = y[i]
|
||||
if t == "knowledge":
|
||||
conf_km[0, 1 if pmw[i] > pk[i] else 0] += 1
|
||||
elif t == "memory_write":
|
||||
conf_km[1, 1 if pmw[i] >= pk[i] else 0] += 1
|
||||
report["kmw"] = {"confusion_p_ordered": conf_km.tolist()}
|
||||
|
||||
# matched pairs with shared subject lexemes, corpus-justified
|
||||
def build_pairs(subject, fam_k, fam_mw):
|
||||
kr = [r for r in pool if r["family_id"] in fam_k]
|
||||
mr = [r for r in pool if r["family_id"] in fam_mw]
|
||||
pairs = []
|
||||
for mw in mr:
|
||||
for k in kr:
|
||||
if subject in mw["n_text"] and subject in k["n_text"]:
|
||||
pairs.append((mw["idx"], k["idx"], mw["n_text"], k["n_text"]))
|
||||
return pairs
|
||||
|
||||
sets = {
|
||||
"water": build_pairs("вод", ["knowledge:recall-fact"], ["fact:water"]),
|
||||
"homelab": build_pairs("dns", ["knowledge:homelab-status"], ["note:homelab"])
|
||||
+ build_pairs("сервер", ["knowledge:homelab-status"], ["note:homelab"])
|
||||
+ build_pairs("vlan", ["knowledge:homelab-status"], ["note:homelab"]),
|
||||
"task": build_pairs("задач", ["knowledge:task-check", "knowledge:deadline"],
|
||||
["note:task"]),
|
||||
}
|
||||
idx_of = {r["idx"]: i for i, r in enumerate(pool)}
|
||||
pair_rep = {}
|
||||
for name, pairs in sets.items():
|
||||
if not pairs:
|
||||
continue
|
||||
ok = 0
|
||||
margins = []
|
||||
bad = []
|
||||
for mi, ki, mx, kx in pairs:
|
||||
mi_i, ki_i = idx_of[mi], idx_of[ki]
|
||||
# MW row should get a higher memory_write probability than the K row
|
||||
mk = (pmw[mi_i] + 0.0)
|
||||
if pmw[mi_i] > pmw[ki_i]:
|
||||
ok += 1
|
||||
else:
|
||||
bad.append((mx[:46], round(float(pmw[mi_i]), 3), kx[:46], round(float(pmw[ki_i]), 3)))
|
||||
margins.append(pmw[mi_i] - pmw[ki_i])
|
||||
pair_rep[name] = {
|
||||
"pairs": len(pairs),
|
||||
"mw_over_k_order_acc": ok / len(pairs),
|
||||
"mean_margin": float(np.mean(margins)),
|
||||
"reversed_examples": bad[:6],
|
||||
}
|
||||
print(f" {name}: pairs={len(pairs)} order_acc={ok/len(pairs):.3f} "
|
||||
f"mean_margin={np.mean(margins):+.3f}")
|
||||
report["kmw"]["matched_pairs"] = pair_rep
|
||||
|
||||
# ── §7 uncertain as explicit class ────────────────────────────────────
|
||||
print("\n§7 uncertain")
|
||||
up = m_best["per_class"]["uncertain"]
|
||||
uc = m_best["confusion"][CLASSES.index("uncertain")]
|
||||
report["uncertain"] = {
|
||||
"per_class": up,
|
||||
"row_from_uncertain": {CLASSES[j]: int(uc[j]) for j in range(5)},
|
||||
"row_to_uncertain": {CLASSES[j]: int(m_best["confusion"][j][CLASSES.index("uncertain")])
|
||||
for j in range(5)},
|
||||
}
|
||||
print(f" uncertain n={up['n']} P={up['p']:.3f} R={up['r']:.3f} F1={up['f1']:.3f}")
|
||||
print(" wrong-→label pulled from uncertain:", report["uncertain"]["row_from_uncertain"])
|
||||
print(" →uncertain pulled from:", report["uncertain"]["row_to_uncertain"])
|
||||
|
||||
# ── §8 OOF confidence / calibration / abstention ─────────────────────
|
||||
print("\n§8 confidence / calibration")
|
||||
conf = p_best.max(1)
|
||||
right = (p_best.argmax(1) == yt)
|
||||
cer = {
|
||||
"correct_conf_mean": float(conf[right].mean()),
|
||||
"correct_conf_median": float(np.median(conf[right])),
|
||||
"wrong_conf_mean": float(conf[~right].mean()),
|
||||
"wrong_conf_median": float(np.median(conf[~right])),
|
||||
"ece": ece(yt, p_best)["ece"],
|
||||
"ece_bins": ece(yt, p_best)["bins"],
|
||||
"log_loss": float(log_loss(yt, p_best, labels=[0, 1, 2, 3, 4])),
|
||||
}
|
||||
# Brier is label-set specific: one-vs-rest mean
|
||||
briers = []
|
||||
for i in range(5):
|
||||
briers.append(brier_score_loss((yt == i).astype(int), p_best[:, i]))
|
||||
cer["brier_macro"] = float(np.mean(briers))
|
||||
report["confidence"] = cer
|
||||
print(f" right conf mean={cer['correct_conf_mean']:.3f} "
|
||||
f"wrong conf mean={cer['wrong_conf_mean']:.3f} ECE={cer['ece']:.4f}")
|
||||
print(f" log_loss={cer['log_loss']:.4f} brier_macro={cer['brier_macro']:.4f}")
|
||||
|
||||
thr_grid = np.linspace(0.10, 0.98, 45)
|
||||
abst = []
|
||||
for t in thr_grid:
|
||||
cov = (conf >= t).mean()
|
||||
if cov == 0:
|
||||
continue
|
||||
keep = conf >= t
|
||||
yt_k = yt[keep]
|
||||
yp_k = p_best[keep].argmax(1)
|
||||
mk_ = cls_metrics(yt_k.tolist(), yp_k.tolist())
|
||||
abst.append({"threshold": round(float(t), 3), "coverage": float(cov),
|
||||
"accuracy": mk_["acc"], "macro_f1": mk_["macro_f1"]})
|
||||
report["confidence"]["abstention_curve"] = abst
|
||||
print(" threshold | coverage | accuracy | macroF1 (first 6/45 + knee)")
|
||||
for row in abst[::9]:
|
||||
print(f" {row['threshold']:.2f} | {row['coverage']:.3f} | "
|
||||
f"{row['accuracy']:.3f} | {row['macro_f1']:.3f}")
|
||||
|
||||
# ── §9 action OOD probes ──────────────────────────────────────────────
|
||||
print("\n§9 action OOD")
|
||||
ood_rows = [r for r in json.load(open(os.path.join(OUT_DIR, "ood.json")))]
|
||||
emb_by_idx = {i: np.asarray(e["embedding"], dtype=np.float64)
|
||||
for i, e in enumerate(slice18_sparse.filter_dev_pool(
|
||||
slice18_sparse.load_data()[1]))}
|
||||
Xo = np.vstack([emb_by_idx[r["idx"]] for r in ood_rows])
|
||||
fold_models = []
|
||||
for te_fold in sorted(set(folds_arr.tolist())):
|
||||
tr = folds_arr != te_fold
|
||||
clf = slice18_sparse.LogisticRegression(
|
||||
C=bestC, max_iter=2000, solver="lbfgs", random_state=42)
|
||||
clf.fit(X[tr], yt[tr])
|
||||
fold_models.append(clf)
|
||||
# XX
|
||||
# comparable to the non-action in-fold behaviour
|
||||
po = np.zeros((len(Xo), 5))
|
||||
for clf in fold_models:
|
||||
po += clf.predict_proba(Xo)
|
||||
po /= len(fold_models)
|
||||
ood_top = int(np.argmax(po.mean(0)))
|
||||
ood_conf = po.max(1)
|
||||
ood_pred = po.argmax(1)
|
||||
top_dist = {CLASSES[i]: int((ood_pred == i).sum()) for i in range(5)}
|
||||
confident_na = int((ood_conf > 0.9).sum())
|
||||
report["ood"] = {
|
||||
"n": len(ood_rows),
|
||||
"top_class": CLASSES[int(ood_top)],
|
||||
"top_class_dist": top_dist,
|
||||
"conf_gt_0.9": confident_na,
|
||||
"conf_gt_0.9_frac": float(confident_na / len(ood_rows)),
|
||||
"conf_mean": float(ood_conf.mean()),
|
||||
"conf_median": float(np.median(ood_conf)),
|
||||
}
|
||||
print(f" action OOD n={len(ood_rows)}: most-confident class={report['ood']['top_class']} "
|
||||
f"dist={top_dist}")
|
||||
print(f" conf>0.9: {confident_na} ({confident_na/len(ood_rows):.3f}) "
|
||||
f"conf mean={report['ood']['conf_mean']:.3f}")
|
||||
|
||||
# ── §10 artifact cost ────────────────────────────────────────────────
|
||||
print("\n§10 artifact")
|
||||
n_params = len(CLASSES) * X.shape[1] + len(CLASSES)
|
||||
fp32 = n_params * 4
|
||||
report["artifact"] = {
|
||||
"e5_dim": X.shape[1],
|
||||
"head_params": n_params,
|
||||
"head_fp32_bytes": fp32,
|
||||
"head_fp32_kib": fp32 / 1024,
|
||||
"head_int8_bytes": n_params,
|
||||
}
|
||||
# incremental latency of the linear head over a batch of 1 (µs)
|
||||
clf = slice18_sparse.LogisticRegression(C=bestC, max_iter=2000, solver="lbfgs", random_state=42)
|
||||
clf.fit(X, yt)
|
||||
x1 = X[:1]
|
||||
for _ in range(50):
|
||||
clf.predict_proba(x1)
|
||||
lat = []
|
||||
for _ in range(2000):
|
||||
t0 = time.perf_counter_ns()
|
||||
clf.predict_proba(x1)
|
||||
lat.append((time.perf_counter_ns() - t0) / 1e3)
|
||||
lat = np.array(lat)
|
||||
report["artifact"]["head_latency_us_mean"] = float(lat.mean())
|
||||
report["artifact"]["head_latency_us_p50"] = float(np.median(lat))
|
||||
print(f" head params={n_params} fp32={fp32/1024:.2f}KiB "
|
||||
f"lat mean={lat.mean():.2f}us p50={np.median(lat):.2f}us")
|
||||
|
||||
with open(os.path.join(OUT_DIR, "results.json"), "w") as f:
|
||||
json.dump(report, f, ensure_ascii=False, indent=1, default=float)
|
||||
print(f"\nwrote {OUT_DIR}/results.json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user