router/semantic: slice 23 corpus fast-path reconciliation — DeriveFastPath over the real router replaces the regex mirror, factory/merge validated, corpus rebuilt (dataset_hash unchanged), drift diagnostic and rerun tooling

cmd/semantic-router-experiment/slice22/heads_main.go

@@ -5,6 +5,7 @@ import (
 	"os"
 
 	"github.com/kami/maven/internal/router"
+	"github.com/kami/maven/internal/router/semantic"
 )
 
 // headsMain runs the deployed cascade minus the resident LLM: stage-0
@@ -48,7 +49,7 @@ func headsMain(poolPath, outPath string) {
 	}
 	defer heads.Close()
 
-	acts := router.DefaultActMatcher{Fns: actVerbList()}
+	acts := router.DefaultActMatcher{Fns: semantic.ExperimentActVerbs()}
 	r := router.New(router.Config{
 		Grammars:   router.StageZeroGrammars(acts),
 		Classifier: cls,

cmd/semantic-router-experiment/slice22/legacy_build.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/kami/maven/internal/router"
+	"github.com/kami/maven/internal/router/semantic"
 )
 
 // Seed loading replicated from cmd/mavend/voicewire.go (seedClassifier) and
@@ -23,26 +24,13 @@ var seedIntents = []router.Intent{
 	router.IntentNote, router.IntentQuery, router.IntentChat, router.IntentSystem,
 }
 
-// actVerbList mirrors the eval fixture's static allowlist
-// (internal/router/semantic/helpers_test.go). The production matcher's
-// allowlist is the deployment's enabled tools; an act grammar can only catch
-// a row whose first tokens match an allowlisted verb, and the experiment's
-// act allowlist is the one the accepted slice-21 methodology used.
-func actVerbList() []string {
-	return []string{
-		"перезапусти", "перезагрузи", "выключи", "включи", "останови", "запусти",
-		"закрой", "открой", "сделай", "поставь",
-		"restart", "reboot", "stop", "start", "turn off", "turn on", "open", "close",
-	}
-}
-
 // buildMinimalRouter reproduces internal/router/semantic/buildMinimalRouter:
 // the daemon's grammar set, a hash-embedder classifier seeded from
 // models/seeds, and the deployed 0.55 threshold. Deterministic and
 // reproducible. The ONNX embedder and the routing heads score elsewhere;
 // this is the floor the eval fixture reports as the legacy baseline.
 func buildMinimalRouter() *router.Router {
-	acts := router.DefaultActMatcher{Fns: actVerbList()}
+	acts := router.DefaultActMatcher{Fns: semantic.ExperimentActVerbs()}
 	cls := router.NewClassifier(router.NewHashEmbedder(1024))
 	seedClassifier(cls)
 	return router.New(router.Config{

cmd/semantic-router-experiment/slice23/main.go

@@ -0,0 +1,243 @@
+// slice23 — fast-path metadata reconciliation diagnostic.
+//
+// Classifies every disagreement between the corpus's stored fast_path_resolved
+// flag and what the production fast path derives today (TryFastPath over the
+// stage-0 grammars with the experiment's act allowlist). Outputs a JSON
+// decomposition and a console summary for docs/evals reports.
+//
+// Usage:
+//
+//	go run ./cmd/semantic-router-experiment/slice23/ -out /tmp/mvn-s23/drift.json
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"sort"
+
+	"github.com/kami/maven/internal/router"
+	"github.com/kami/maven/internal/router/semantic"
+)
+
+type conflict struct {
+	Text     string `json:"text"`
+	SourceID string `json:"source_id"`
+	Route    string `json:"route"`
+	Source   string `json:"source"`
+	Group    string `json:"split_group"`
+	Tags     []string `json:"tags,omitempty"`
+	Dev      bool    `json:"dev"`
+	// Direction: claimed_fast_now_miss = stored fast, runtime residual;
+	// mirror_missed = stored residual, runtime fast.
+	Direction string `json:"direction"`
+	Grammar   string `json:"grammar,omitempty"`
+	// ShapeDeclined is true when at least one stage-0 grammar matched the
+	// utterance's shape but refused the content (falls through like the router).
+	ShapeDeclined bool `json:"shape_declined"`
+	// DeclinedGrammars names every stage-0 grammar that matched the shape but
+	// declined the content, for claimed_fast_now_miss rows.
+	DeclinedGrammars []string `json:"declined_grammars,omitempty"`
+}
+
+type report struct {
+	Meta       metaSummary `json:"meta"`
+	Pop        popSummary  `json:"population"`
+	Conflicts  []conflict  `json:"conflicts"`
+	ByRoute    map[string]map[string]int `json:"by_route"`
+	ByGrammar  map[string]int `json:"by_grammar"`
+	BySource   map[string]map[string]int `json:"by_source"`
+	ByFamily   map[string]map[string]int `json:"by_family"`
+	Direction  map[string]int `json:"by_direction"`
+	Declined   int `json:"claimed_fast_with_declined_shape"`
+	NoShape    int `json:"claimed_fast_with_no_shape"`
+}
+
+type metaSummary struct {
+	Total    int `json:"total"`
+	DevCount int `json:"dev_count"`
+	Frozen   int `json:"frozen_count"`
+}
+
+type popSummary struct {
+	StoredFast   int `json:"stored_fast"`
+	StoredResid  int `json:"stored_residual"`
+	DerivedFast  int `json:"derived_fast"`
+	DerivedResid int `json:"derived_residual"`
+	// Dev-pool residual route counts derived as the router sees them today.
+	DevResidualByRoute map[string]int `json:"dev_residual_by_route"`
+	// Dev-pool residual non-action + action OOD as derived.
+	DevResidualNonAction int `json:"dev_residual_non_action"`
+	DevResidualAction    int `json:"dev_residual_action"`
+	// Fast rows in the dev pool, derived.
+	DevFast int `json:"dev_fast"`
+}
+
+func main() {
+	outPath := flag.String("out", "/tmp/mvn-s23/drift.json", "output JSON path")
+	flag.Parse()
+
+	exs, err := semantic.LoadCorpus()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "load corpus: %v\n", err)
+		os.Exit(1)
+	}
+
+	_, dev, _ := semantic.FrozenHoldoutSplit(exs)
+	devSet := make(map[string]bool, len(dev))
+	for _, e := range dev {
+		devSet[e.SourceID] = true
+	}
+
+	// Stage-zero grammar list for shape/declined attribution (same list the
+	// derivation walks).
+	acts := router.DefaultActMatcher{Fns: semantic.ExperimentActVerbs()}
+	gs := router.StageZeroGrammars(acts)
+
+	var (
+		conflicts      []conflict
+		byRoute        = map[string]map[string]int{}
+		byGrammar      = map[string]int{}
+		bySource       = map[string]map[string]int{}
+		byFamily       = map[string]map[string]int{}
+		byDirection    = map[string]int{}
+		storedFast, derivedFast, declined, noShape int
+		devResidByRoute = map[string]int{}
+		devResidNonAct, devResidAct, devFast int
+	)
+
+	for _, e := range exs {
+		o := semantic.DeriveFastPath(e.Text)
+		inDev := devSet[e.SourceID]
+
+		st := e.FastPathResolved
+		if st {
+			storedFast++
+		}
+		if o.Matched {
+			derivedFast++
+		}
+
+		var c *conflict
+		switch {
+		case st && o.Matched:
+		case st && !o.Matched:
+			// Stored fast but the runtime misses. Attribute why.
+			shapeDeclined := false
+			var declinedNames []string
+			for _, g := range gs {
+				_, matched, ok := g.Evaluate(e.Text)
+				if matched && !ok {
+					shapeDeclined = true
+					declinedNames = append(declinedNames, g.Name)
+				}
+			}
+			if shapeDeclined {
+				declined++
+			} else {
+				noShape++
+			}
+			c = &conflict{Direction: "claimed_fast_now_miss", ShapeDeclined: shapeDeclined, DeclinedGrammars: declinedNames}
+		case !st && o.Matched:
+			c = &conflict{Direction: "mirror_missed", Grammar: o.Grammar}
+		}
+
+		if c != nil {
+			c.Text = e.Text
+			c.SourceID = e.SourceID
+			c.Route = string(e.Route)
+			c.Source = e.Source
+			c.Group = e.SplitGroup
+			c.Tags = e.Tags
+			c.Dev = inDev
+			conflicts = append(conflicts, *c)
+			byDirection[c.Direction]++
+			byGrammar[c.Grammar]++
+			if byRoute[c.Direction] == nil {
+				byRoute[c.Direction] = map[string]int{}
+			}
+			byRoute[c.Direction][c.Route]++
+			if bySource[c.Direction] == nil {
+				bySource[c.Direction] = map[string]int{}
+			}
+			bySource[c.Direction][c.Source]++
+			if byFamily[c.Direction] == nil {
+				byFamily[c.Direction] = map[string]int{}
+			}
+			byFamily[c.Direction][c.Group]++
+		}
+
+		if inDev {
+			if o.Matched {
+				devFast++
+			} else {
+				devResidByRoute[string(e.Route)]++
+				if e.Route == semantic.RouteAction {
+					devResidAct++
+				} else {
+					devResidNonAct++
+				}
+			}
+		}
+	}
+
+	sort.Slice(conflicts, func(i, j int) bool { return conflicts[i].SourceID < conflicts[j].SourceID })
+
+	rep := report{
+		Meta: metaSummary{Total: len(exs), DevCount: len(dev), Frozen: len(exs) - len(dev)},
+		Pop: popSummary{
+			StoredFast: storedFast, StoredResid: len(exs) - storedFast,
+			DerivedFast: derivedFast, DerivedResid: len(exs) - derivedFast,
+			DevResidualByRoute: devResidByRoute,
+			DevResidualNonAction: devResidNonAct, DevResidualAction: devResidAct,
+			DevFast: devFast,
+		},
+		Conflicts: conflicts,
+		ByRoute:   byRoute, ByGrammar: byGrammar, BySource: bySource, ByFamily: byFamily,
+		Direction: byDirection, Declined: declined, NoShape: noShape,
+	}
+
+	data, err := json.MarshalIndent(rep, "", "  ")
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "marshal: %v\n", err)
+		os.Exit(1)
+	}
+	if err := os.WriteFile(*outPath, data, 0644); err != nil {
+		fmt.Fprintf(os.Stderr, "write %s: %v\n", *outPath, err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("total %d (dev %d, frozen %d)\n", rep.Meta.Total, rep.Meta.DevCount, rep.Meta.Frozen)
+	fmt.Printf("stored  fast=%d residual=%d\n", rep.Pop.StoredFast, rep.Pop.StoredResid)
+	fmt.Printf("derived fast=%d residual=%d\n", rep.Pop.DerivedFast, rep.Pop.DerivedResid)
+	fmt.Printf("disagreements total %d\n", len(conflicts))
+	for _, d := range []string{"claimed_fast_now_miss", "mirror_missed"} {
+		fmt.Printf("  %-22s %d\n", d, byDirection[d])
+		if d == "claimed_fast_now_miss" {
+			fmt.Printf("    with declined shape: %d   no shape: %d\n", declined, noShape)
+		}
+	}
+	fmt.Println("  mirror-missed by grammar:")
+	for _, k := range sortedKeys(byGrammar) {
+		fmt.Printf("    %-28s %d\n", k, byGrammar[k])
+	}
+	fmt.Println("  by route:")
+	for _, d := range sortedKeys(byRoute) {
+		fmt.Printf("    %-22s %v\n", d, byRoute[d])
+	}
+	fmt.Printf("dev pool derived: fast=%d residual=%d (non-action=%d action=%d)\n",
+		rep.Pop.DevFast, rep.Pop.DevResidualNonAction+rep.Pop.DevResidualAction,
+		rep.Pop.DevResidualNonAction, rep.Pop.DevResidualAction)
+	fmt.Printf("dev residual by route: %v\n", rep.Pop.DevResidualByRoute)
+	fmt.Printf("wrote %s\n", *outPath)
+}
+
+func sortedKeys[T any](m map[string]T) []string {
+	ks := make([]string, 0, len(m))
+	for k := range m {
+		ks = append(ks, k)
+	}
+	sort.Strings(ks)
+	return ks
+}

cmd/semantic-router-experiment/slice23_emit.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+"""
+Slice 23 emit: five-way residual non-action semantic router — data files
+========================================================================
+
+Slice 23 reconciles corpus fast-path metadata with the production router
+(TryFastPath over stage-0 grammars). The corpus builder no longer mirrors the
+grammars by hand; fast_path_resolved is derived from the router, so this emit
+flags exactly the rows the router genuinely leaves for the general cascade.
+
+This script only repackages the frozen dev pool for the Go legacy baseline and
+the Python experiment, writing into /tmp/mvn-s23 so the slice-22 artifacts
+stay untouched. Logic is slice22_emit.py verbatim; only OUT_DIR differs.
+
+    /tmp/mvn-s23/pool.json    residual non-action dev rows: idx, text, n_text,
+                              route, tags, cv_fold, split_group, family_id,
+                              source_id  (1509 rows)
+    /tmp/mvn-s23/ood.json     residual ACTION dev rows (720): same shape; OOD
+                              probes only, never primary metrics
+    /tmp/mvn-s23/stats.json   population summary (routes, families, folds)
+"""
+
+import json
+import os
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, HERE)
+
+import slice18_sparse  # noqa: E402  (normalize_match_text, load_data, filters)
+
+OUT_DIR = "/tmp/mvn-s23"
+
+ROUTES = ["conversation", "knowledge", "memory_write", "system", "uncertain"]
+
+
+def main():
+    meta, examples = slice18_sparse.load_data()
+    dev = slice18_sparse.filter_dev_pool(examples)
+    print(f"dev pool: {len(dev)} rows "
+          f"(meta declares dev_count={meta.get('dev_count')})")
+
+    rows = []
+    for i, e in enumerate(dev):
+        if not e["fast_path_resolved"]:
+            rows.append({
+                "idx": i,
+                "text": e["text"],
+                "n_text": slice18_sparse.normalize_match_text(e["text"]),
+                "route": e["route"],
+                "tags": sorted(set(e.get("tags", []))),
+                "cv_fold": e["cv_fold"],
+                "split_group": e["split_group"],
+                "family_id": e["family_id"],
+                "source_id": e["source_id"],
+            })
+
+    na = [r for r in rows if r["route"] != "action"]
+    ood = [r for r in rows if r["route"] == "action"]
+    print(f"residual rows: {len(rows)}  non-action: {len(na)}  action(OOD): {len(ood)}")
+
+    by_route = {}
+    for r in na:
+        by_route[r["route"]] = by_route.get(r["route"], 0) + 1
+    print("routes:", by_route)
+    assert sum(by_route.values()) == len(na)
+    assert set(ROUTES) == set(by_route), "route set must be the five-way"
+
+    by_family = {}
+    for r in na:
+        by_family[r["family_id"]] = by_family.get(r["family_id"], 0) + 1
+    by_fold = {}
+    for r in na:
+        by_fold[r["cv_fold"]] = by_fold.get(r["cv_fold"], 0) + 1
+    print(f"family_ids: {len(by_family)}  split_groups: {len(set(r['split_group'] for r in na))}")
+    print("folds:", by_fold)
+
+    os.makedirs(OUT_DIR, exist_ok=True)
+    with open(os.path.join(OUT_DIR, "pool.json"), "w") as f:
+        json.dump(na, f, ensure_ascii=False, indent=1)
+    with open(os.path.join(OUT_DIR, "ood.json"), "w") as f:
+        json.dump(ood, f, ensure_ascii=False, indent=1)
+    with open(os.path.join(OUT_DIR, "stats.json"), "w") as f:
+        json.dump({
+            "dev_count": len(dev),
+            "residual_count": len(rows),
+            "non_action_count": len(na),
+            "action_ood_count": len(ood),
+            "routes": by_route,
+            "family_ids": len(by_family),
+            "folds": by_fold,
+            "top_family": dict(sorted(by_family.items(), key=lambda kv: -kv[1])[:15]),
+        }, f, ensure_ascii=False, indent=1)
+    print(f"wrote {OUT_DIR}/{{pool,ood,stats}}.json")
+
+
+if __name__ == "__main__":
+    main()

cmd/semantic-router-experiment/slice23_main.py

@@ -0,0 +1,559 @@
+#!/usr/bin/env python3
+"""
+Slice 23: five-way residual non-action semantic router — corrected population
+============================================================================
+
+Slice 22 reported the corpus's fast-path mirror was stale next to the
+production stage-0 grammars (185 residual rows resolved at runtime). Slice 23
+derives fast_path_resolved from the real router, rebuilds the corpus, and
+re-measures the primary slice-22 results on the corrected residual pool.
+Logic and configs are slice22_main.py verbatim; only OUT_DIR differs.
+
+Population: the corrected dev-pool residual non-action rows (1509; the pool
+written by slice23_emit.py). Corrected action rows (720) are OOD probes only.
+
+Metrics written to /tmp/mvn-s23/results.json:
+  §1 population
+  §2 legacy baseline (legacy.json / legacy_heads.json): acc, macro-F1,
+     per-class P/R/F1, confusion, illegal_action_prediction count
+  §3 e5-linear primary head: C grid, grouped CV OOF, per-fold P/R/F1 +
+     variance + composition
+  §4 floors: majority, centroid (cosine nearest-mean), sparse word+char
+     TF-IDF logistic (slice18 builder), all grouped CV
+  §5 route-family (family_id) leave-family-out
+  §6 knowledge vs memory_write: matched pairs (water/homelab/task) ordering
+  §7 uncertain as an explicit class: P/R/F1 + top confusions
+  §8 OOF confidence: max-softmax correct/wrong, ECE, log-loss, Brier,
+     coverage/accuracy/macro-F1 abstention curves (no threshold chosen)
+  §9 action OOD probes: fold models applied to the corrected action rows
+  §10 artifact cost: head params, serialized bytes, incremental head latency
+
+No corpus label is changed. No frozen-holdout rows are inspected.
+"""
+
+import json
+import os
+import sys
+import time
+
+import numpy as np
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, HERE)
+
+import slice18_sparse  # noqa: E402
+import slice19_main    # noqa: E402
+
+EMB_PATH = "/tmp/mvn-experiment/embeddings.json"
+OUT_DIR = "/tmp/mvn-s23"
+CLASSES = ["conversation", "knowledge", "memory_write", "system", "uncertain"]
+CLASS_PREFIX = ["conversation", "knowledge", "memory_write", "system", "uncertain"]
+C_GRID = [0.1, 1.0, 10.0]
+
+# Route-family holdouts the report calls out by name (slice-22 brief): every
+# family that is not part of the shared subject inventory on either side.
+HOLDOUT_GROUPS = {
+    "capability": ["knowledge:capability-ha", "knowledge:capability-tool"],
+    "world": ["knowledge:world-def", "knowledge:world-explain"],
+    "calendar": ["knowledge:calendar", "knowledge:calendar-time", "knowledge:calendar-next"],
+    "recall": ["knowledge:recall-fact", "knowledge:recall-note", "knowledge:recall-possessive"],
+    "fact": ["fact:meal", "fact:water", "fact:sleep", "fact:shower", "fact:break", "fact:pills", "fact:exercise"],
+    "note": ["note:idea", "note:homelab", "note:task"],
+    "remember": ["free:remember"],
+    "system": None,   # all system:*
+    "conversation": None,
+    "uncertain": None,
+}
+
+
+def load_pool_and_embeds():
+    with open(os.path.join(OUT_DIR, "pool.json")) as f:
+        pool = json.load(f)
+    meta, examples = slice18_sparse.load_data()
+    dev = slice18_sparse.filter_dev_pool(examples)
+    by_idx = {e["dev_idx"]: e for e in dev} if "dev_idx" in dev[0] else None
+    # pool rows carry idx = position among dev_pool rows in dev order
+    emb_by_idx = {i: np.asarray(e["embedding"], dtype=np.float64)
+                  for i, e in enumerate(dev)}
+    for r in pool:
+        r["emb"] = emb_by_idx[r["idx"]]
+        r["y"] = r["route"]
+    return pool, meta
+
+
+def oof_proba_grouped(X, y, folds, C=1.0):
+    """Grouped OOF probability matrix (n×5, class order CLASSES)."""
+    y_idx = np.array([CLASSES.index(c) for c in y])
+    folds = np.asarray(folds)
+    proba = np.zeros((len(y_idx), len(CLASSES)))
+    for te_fold in sorted(set(folds.tolist())):
+        tr = folds != te_fold
+        te = folds == te_fold
+        clf = slice18_sparse.LogisticRegression(
+            C=C, max_iter=2000, solver="lbfgs", random_state=42)
+        clf.fit(X[tr], y_idx[tr])
+        proba[te] = clf.predict_proba(X[te])
+    return proba
+
+
+def cls_metrics(yt, yp):
+    import sklearn.metrics as m
+    yt = np.asarray(yt)
+    yp = np.asarray(yp)
+    if yt.dtype != np.int64 and yt.dtype != np.int32:
+        yt = np.array([CLASSES.index(c) for c in yt])
+    if yp.dtype != np.int64 and yp.dtype != np.int32:
+        yp = np.array([CLASSES.index(c) for c in yp])
+    labels = list(range(len(CLASSES)))
+    n = len(yt)
+    acc = m.accuracy_score(yt, yp)
+    macro = m.f1_score(yt, yp, average="macro", labels=labels, zero_division=0)
+    pr, rc, f1, sup = m.precision_recall_fscore_support(
+        yt, yp, labels=labels, zero_division=0)
+    per = {c: {"p": float(pr[i]), "r": float(rc[i]), "f1": float(f1[i]), "n": int(sup[i])}
+           for i, c in enumerate(CLASSES)}
+    conf = m.confusion_matrix(yt, yp, labels=labels).tolist()
+    return {"n": n, "acc": acc, "macro_f1": macro, "per_class": per, "confusion": conf}
+
+
+def fold_report(yt, proba, folds, true_y):
+    out = {}
+    folds_arr = np.asarray(folds)
+    comp = {}
+    for f in sorted(set(folds_arr.tolist())):
+        mask = folds_arr == f
+        yt_f = [CLASSES.index(y) for y in true_y[mask]]
+        comp[f] = {c: int((np.array(true_y[mask]) == c).sum()) for c in CLASSES}
+    per_fold = {}
+    for f in sorted(set(folds_arr.tolist())):
+        mask = folds_arr == f
+        yp = proba[mask].argmax(1).tolist()
+        m = cls_metrics([yt[i] for i in np.where(mask)[0].tolist()], yp)
+        per_fold[f] = {"acc": m["acc"], "macro_f1": m["macro_f1"]}
+    out["composition"] = comp
+    out["per_fold"] = per_fold
+    accs = [v["acc"] for v in per_fold.values()]
+    macros = [v["macro_f1"] for v in per_fold.values()]
+    out["acc_mean"] = float(np.mean(accs))
+    out["acc_std"] = float(np.std(accs))
+    out["macro_f1_mean"] = float(np.mean(macros))
+    out["macro_f1_std"] = float(np.std(macros))
+    return out
+
+
+def ece(yt, proba, n_bins=15):
+    conf = proba.max(1)
+    pred = proba.argmax(1)
+    acc = (pred == yt).astype(float)
+    bins = np.linspace(0, 1, n_bins + 1)
+    tot = 0.0
+    details = []
+    counts = 0
+    for i in range(n_bins):
+        lo, hi = bins[i], bins[i + 1]
+        m = (conf >= lo) & (conf < hi) if i < n_bins - 1 else conf >= lo
+        if m.sum() == 0:
+            continue
+        acc_m = acc[m].mean()
+        conf_m = conf[m].mean()
+        w = m.sum() / len(conf)
+        tot += w * abs(acc_m - conf_m)
+        counts += int(m.sum())
+        details.append({"bin": i, "lo": lo, "hi": hi, "conf": float(conf_m),
+                        "acc": float(acc_m), "n": int(m.sum())})
+    return {"ece": float(tot), "n_bins": n_bins, "counted": counts, "bins": details}
+
+
+def main():
+    pool, meta = load_pool_and_embeds()
+    pool.sort(key=lambda r: r["idx"])
+    print(f"pool: {len(pool)} rows")
+
+    from sklearn.metrics import brier_score_loss, log_loss
+
+    report = {"population": {}, "legacy": {}, "e5_linear": {}, "floors": {},
+              "family_holdouts": {}, "kmw": {}, "uncertain": {}, "confidence": {},
+              "ood": {}, "artifact": {}}
+
+    # ── §1 population ──────────────────────────────────────────────────────
+    cnt = {}
+    for r in pool:
+        cnt[r["y"]] = cnt.get(r["y"], 0) + 1
+    report["population"] = {
+        "n": len(pool),
+        "routes": cnt,
+        "family_ids": len(set(r["family_id"] for r in pool)),
+        "split_groups": len(set(r["split_group"] for r in pool)),
+        "folds": {str(f): int(sum(1 for r in pool if r["cv_fold"] == f)) for f in sorted(set(r["cv_fold"] for r in pool))},
+        "corpus": {k: v for k, v in meta.items() if k in
+                   ("dev_count", "residual_count", "fast_path_count",
+                    "dimension", "embedder_id", "input_template", "pooling", "normalization")},
+    }
+    print("\n§1 population:", report["population"])
+
+    X = np.vstack([r["emb"] for r in pool])
+    y = np.array([r["y"] for r in pool])
+    folds = np.array([r["cv_fold"] for r in pool])
+    yt = np.array([CLASSES.index(c) for c in y])
+
+    # ── §2 legacy baselines ────────────────────────────────────────────────
+    import collections
+    for tag, fname in [("hash", "legacy.json"), ("heads", "legacy_heads.json")]:
+        path = os.path.join(OUT_DIR, fname)
+        if not os.path.exists(path):
+            continue
+        leg = json.load(open(path))
+        leg_by_idx = {r["idx"]: r for r in leg}
+        yp_leg = []
+        illegal = []
+        for r in pool:
+            lr = leg_by_idx[r["idx"]]
+            if lr["illegal_action_prediction"]:
+                illegal.append(lr)
+                yp_leg.append("action")
+            else:
+                yp_leg.append(lr["class"])
+        yp_leg = np.array(yp_leg)
+        # five-way: an 'action' prediction is an error (outside the label set)
+        yp5 = np.array([("uncertain" if p == "action" else p) for p in yp_leg])
+        m = cls_metrics(y, yp5)
+        m["illegal_action_prediction"] = len(illegal)
+        m["illegal_cases"] = [{"idx": i["idx"], "text": i["text"], "route": i["route"],
+                               "intent": i["intent"], "producer": i["producer"],
+                               "confidence": i["confidence"]} for i in illegal]
+        # per-cell confusion also shows 'action' column
+        conf_counts = collections.Counter(zip(y, yp_leg))
+        m["confusion_with_action"] = {f"{a}->{b}": int(c) for (a, b), c in conf_counts.items()}
+        report["legacy"][tag] = m
+        print(f"\n§2 legacy ({tag}) acc={m['acc']:.4f} macroF1={m['macro_f1']:.4f} "
+              f"illegal={len(illegal)}")
+        for c in CLASSES:
+            p = m["per_class"][c]
+            print(f"   {c:<14} P={p['p']:.3f} R={p['r']:.3f} F1={p['f1']:.3f} n={p['n']}")
+
+    # grammar-pure residual: rows not resolved by any current stage-0 grammar
+    if os.path.exists(os.path.join(OUT_DIR, "legacy.json")):
+        leg = json.load(open(os.path.join(OUT_DIR, "legacy.json")))
+        gh = {r["idx"] for r in leg if r["producer"] == "grammar"}
+        gp_mask = np.array([r["idx"] not in gh for r in pool])
+        report["grammar_drift"] = {
+            "grammar_hits_in_pool": len(gh),
+            "grammar_pure_n": int(gp_mask.sum()),
+        }
+
+    # ── §3 e5-linear primary head ─────────────────────────────────────────
+    print("\n§3 e5-linear")
+    bestC, bestMac = 1.0, -1.0
+    grid = {}
+    for C in C_GRID:
+        p = oof_proba_grouped(X, y, folds, C=C)
+        mp = cls_metrics(y, p.argmax(1).tolist())
+        grid[float(C)] = {"acc": mp["acc"], "macro_f1": mp["macro_f1"]}
+        print(f"   C={C}  acc={mp['acc']:.4f} macroF1={mp['macro_f1']:.4f}")
+        if mp["macro_f1"] > bestMac:
+            bestMac, bestC = mp["macro_f1"], C
+    print(f"   -> best C={bestC}")
+    p_best = oof_proba_grouped(X, y, folds, C=bestC)
+    m_best = cls_metrics(y, p_best.argmax(1).tolist())
+    m_best["C"] = bestC
+    m_best["C_grid"] = grid
+    m_best["folds"] = fold_report(yt, p_best, folds, y)
+    report["e5_linear"] = m_best
+    for f, v in m_best["folds"]["per_fold"].items():
+        print(f"   fold {f}: acc={v['acc']:.4f} macroF1={v['macro_f1']:.4f}")
+    print(f"   fold acc mean={m_best['folds']['acc_mean']:.4f} "
+          f"std={m_best['folds']['acc_std']:.4f}; "
+          f"macroF1 mean={m_best['folds']['macro_f1_mean']:.4f} "
+          f"std={m_best['folds']['macro_f1_std']:.4f}")
+    for c in CLASSES:
+        p_ = m_best["per_class"][c]
+        print(f"   {c:<14} P={p_['p']:.3f} R={p_['r']:.3f} F1={p_['f1']:.3f} n={p_['n']}")
+
+    # grammar-pure sensitivity for the primary head
+    if "grammar_drift" in report:
+        mp_gp = cls_metrics(y[gp_mask], p_best[gp_mask].argmax(1).tolist())
+        report["e5_linear"]["grammar_pure"] = {
+            "acc": mp_gp["acc"], "macro_f1": mp_gp["macro_f1"], "n": int(gp_mask.sum())}
+
+    # ── §4 floors ─────────────────────────────────────────────────────────
+    print("\n§4 floors")
+    # majority floor
+    maj = CLASSES.index("knowledge")
+    ym = np.full(len(y), maj)
+    mm = cls_metrics(y, ym)
+    report["floors"]["majority"] = {"acc": mm["acc"], "macro_f1": mm["macro_f1"],
+                                    "per_class": mm["per_class"]}
+    print(f"   majority (predict {CLASSES[maj]}): acc={mm['acc']:.4f} macroF1={mm['macro_f1']:.4f}")
+
+    # centroid floor: cosine to per-class mean of the training folds' embeddings
+    cf_proba = np.zeros((len(yt), len(CLASSES)))
+    folds_arr = np.asarray(folds)
+    for te_fold in sorted(set(folds_arr.tolist())):
+        tr = folds_arr != te_fold
+        te = folds_arr == te_fold
+        centroids = []
+        for c in CLASSES:
+            idxs = np.where(tr & (y == c))[0]
+            ctr = X[idxs].mean(axis=0)
+            ctr = ctr / np.linalg.norm(ctr)
+            centroids.append(ctr)
+        Cm = np.vstack(centroids)
+        sims = X[te] @ Cm.T
+        cf_proba[te] = sims
+    yc = cf_proba.argmax(1)
+    # accuracy + macroF1 with the same 5-way
+    mc = cls_metrics(y, yc.tolist())
+    report["floors"]["centroid"] = {"acc": mc["acc"], "macro_f1": mc["macro_f1"],
+                                    "per_class": mc["per_class"]}
+    print(f"   centroid cosine: acc={mc['acc']:.4f} macroF1={mc['macro_f1']:.4f}")
+
+    # sparse word+char logistic (slice18 builder, grouped CV, five-way)
+    texts = [r["n_text"] for r in pool]
+    Xs, _vec = slice18_sparse.build_features(texts, "both")
+    psp = np.zeros((len(yt), len(CLASSES)))
+    for te_fold in sorted(set(folds_arr.tolist())):
+        tr = folds_arr != te_fold
+        te = folds_arr == te_fold
+        clf = slice18_sparse.LogisticRegression(
+            C=1.0, max_iter=2000, solver="lbfgs", random_state=42)
+        clf.fit(Xs[tr], yt[tr])
+        psp[te] = clf.predict_proba(Xs[te])
+    msp = cls_metrics(y, psp.argmax(1).tolist())
+    report["floors"]["sparse_word_char"] = {
+        "acc": msp["acc"], "macro_f1": msp["macro_f1"], "per_class": msp["per_class"],
+        "vocab": slice18_sparse.vocab_size(_vec)}
+    print(f"   sparse both: acc={msp['acc']:.4f} macroF1={msp['macro_f1']:.4f} "
+          f"vocab={report['floors']['sparse_word_char']['vocab']}")
+
+    # ── §5 route-family holdouts ─────────────────────────────────────────
+    print("\n§5 route-family holdouts")
+    fam = np.array([r["family_id"] for r in pool])
+    holdouts = {}
+    all_fams = sorted(set(fam.tolist()))
+    for grp, fams in HOLDOUT_GROUPS.items():
+        if fams is None:
+            fams = [f for f in all_fams if f.startswith(grp + ":")]
+        mask = np.isin(fam, fams)
+        if mask.sum() == 0:
+            continue
+        tr = ~mask
+        clf = slice18_sparse.LogisticRegression(
+            C=bestC, max_iter=2000, solver="lbfgs", random_state=42)
+        clf.fit(X[tr], yt[tr])
+        ypgrp = clf.predict(X[mask])
+        m = cls_metrics([CLASSES.index(c) for c in y[mask]], ypgrp.tolist())
+        m["families"] = fams
+        m["rows"] = int(mask.sum())
+        holdouts[grp] = {"acc": m["acc"], "macro_f1": m["macro_f1"], "n": int(mask.sum()),
+                         "per_class": m["per_class"]}
+        print(f"   {grp:<14} n={m['rows']} acc={m['acc']:.4f} macroF1={m['macro_f1']:.4f}")
+    # full leave-one-family-out summary
+    lofo_accs = []
+    lofo_f1s = []
+    for f in all_fams:
+        mask = fam == f
+        tr = ~mask
+        clf = slice18_sparse.LogisticRegression(
+            C=bestC, max_iter=2000, solver="lbfgs", random_state=42)
+        clf.fit(X[tr], yt[tr])
+        ypf = clf.predict(X[mask])
+        m = cls_metrics([CLASSES.index(c) for c in y[mask]], ypf.tolist())
+        lofo_accs.append(m["acc"])
+        lofo_f1s.append(m["macro_f1"])
+    holdouts["_all_49_lo_"] = {"n_families": len(all_fams),
+                               "acc_mean": float(np.mean(lofo_accs)),
+                               "macro_f1_mean": float(np.mean(lofo_f1s))}
+    report["family_holdouts"] = holdouts
+    print(f"   leave-one-family-out over {len(all_fams)} families: "
+          f"acc mean={np.mean(lofo_accs):.4f} macroF1 mean={np.mean(lofo_f1s):.4f}")
+
+    # ── §6 knowledge vs memory_write ─────────────────────────────────────
+    print("\n§6 knowledge vs memory_write")
+    # reuse e5-linear OOF: does the model put the higher probability on the
+    # right side (memory_write for a write, knowledge for a recall)?
+    conf_km = np.zeros((2, 2))
+    pk = p_best[:, CLASSES.index("knowledge")]
+    pmw = p_best[:, CLASSES.index("memory_write")]
+    for i in range(len(yt)):
+        t = y[i]
+        if t == "knowledge":
+            conf_km[0, 1 if pmw[i] > pk[i] else 0] += 1
+        elif t == "memory_write":
+            conf_km[1, 1 if pmw[i] >= pk[i] else 0] += 1
+    report["kmw"] = {"confusion_p_ordered": conf_km.tolist()}
+
+    # matched pairs with shared subject lexemes, corpus-justified
+    def build_pairs(subject, fam_k, fam_mw):
+        kr = [r for r in pool if r["family_id"] in fam_k]
+        mr = [r for r in pool if r["family_id"] in fam_mw]
+        pairs = []
+        for mw in mr:
+            for k in kr:
+                if subject in mw["n_text"] and subject in k["n_text"]:
+                    pairs.append((mw["idx"], k["idx"], mw["n_text"], k["n_text"]))
+        return pairs
+
+    sets = {
+        "water":  build_pairs("вод", ["knowledge:recall-fact"], ["fact:water"]),
+        "homelab": build_pairs("dns", ["knowledge:homelab-status"], ["note:homelab"])
+                   + build_pairs("сервер", ["knowledge:homelab-status"], ["note:homelab"])
+                   + build_pairs("vlan", ["knowledge:homelab-status"], ["note:homelab"]),
+        "task":   build_pairs("задач", ["knowledge:task-check", "knowledge:deadline"],
+                              ["note:task"]),
+    }
+    idx_of = {r["idx"]: i for i, r in enumerate(pool)}
+    pair_rep = {}
+    for name, pairs in sets.items():
+        if not pairs:
+            continue
+        ok = 0
+        margins = []
+        bad = []
+        for mi, ki, mx, kx in pairs:
+            mi_i, ki_i = idx_of[mi], idx_of[ki]
+            # MW row should get a higher memory_write probability than the K row
+            mk = (pmw[mi_i] + 0.0)
+            if pmw[mi_i] > pmw[ki_i]:
+                ok += 1
+            else:
+                bad.append((mx[:46], round(float(pmw[mi_i]), 3), kx[:46], round(float(pmw[ki_i]), 3)))
+            margins.append(pmw[mi_i] - pmw[ki_i])
+        pair_rep[name] = {
+            "pairs": len(pairs),
+            "mw_over_k_order_acc": ok / len(pairs),
+            "mean_margin": float(np.mean(margins)),
+            "reversed_examples": bad[:6],
+        }
+        print(f"   {name}: pairs={len(pairs)}  order_acc={ok/len(pairs):.3f} "
+              f"mean_margin={np.mean(margins):+.3f}")
+    report["kmw"]["matched_pairs"] = pair_rep
+
+    # ── §7 uncertain as explicit class ────────────────────────────────────
+    print("\n§7 uncertain")
+    up = m_best["per_class"]["uncertain"]
+    uc = m_best["confusion"][CLASSES.index("uncertain")]
+    report["uncertain"] = {
+        "per_class": up,
+        "row_from_uncertain": {CLASSES[j]: int(uc[j]) for j in range(5)},
+        "row_to_uncertain": {CLASSES[j]: int(m_best["confusion"][j][CLASSES.index("uncertain")])
+                             for j in range(5)},
+    }
+    print(f"   uncertain n={up['n']} P={up['p']:.3f} R={up['r']:.3f} F1={up['f1']:.3f}")
+    print("   wrong-→label pulled from uncertain:", report["uncertain"]["row_from_uncertain"])
+    print("   →uncertain pulled from:", report["uncertain"]["row_to_uncertain"])
+
+    # ── §8 OOF confidence / calibration / abstention ─────────────────────
+    print("\n§8 confidence / calibration")
+    conf = p_best.max(1)
+    right = (p_best.argmax(1) == yt)
+    cer = {
+        "correct_conf_mean": float(conf[right].mean()),
+        "correct_conf_median": float(np.median(conf[right])),
+        "wrong_conf_mean": float(conf[~right].mean()),
+        "wrong_conf_median": float(np.median(conf[~right])),
+        "ece": ece(yt, p_best)["ece"],
+        "ece_bins": ece(yt, p_best)["bins"],
+        "log_loss": float(log_loss(yt, p_best, labels=[0, 1, 2, 3, 4])),
+    }
+    # Brier is label-set specific: one-vs-rest mean
+    briers = []
+    for i in range(5):
+        briers.append(brier_score_loss((yt == i).astype(int), p_best[:, i]))
+    cer["brier_macro"] = float(np.mean(briers))
+    report["confidence"] = cer
+    print(f"   right conf mean={cer['correct_conf_mean']:.3f} "
+          f"wrong conf mean={cer['wrong_conf_mean']:.3f} ECE={cer['ece']:.4f}")
+    print(f"   log_loss={cer['log_loss']:.4f} brier_macro={cer['brier_macro']:.4f}")
+
+    thr_grid = np.linspace(0.10, 0.98, 45)
+    abst = []
+    for t in thr_grid:
+        cov = (conf >= t).mean()
+        if cov == 0:
+            continue
+        keep = conf >= t
+        yt_k = yt[keep]
+        yp_k = p_best[keep].argmax(1)
+        mk_ = cls_metrics(yt_k.tolist(), yp_k.tolist())
+        abst.append({"threshold": round(float(t), 3), "coverage": float(cov),
+                     "accuracy": mk_["acc"], "macro_f1": mk_["macro_f1"]})
+    report["confidence"]["abstention_curve"] = abst
+    print("   threshold | coverage | accuracy | macroF1 (first 6/45 + knee)")
+    for row in abst[::9]:
+        print(f"     {row['threshold']:.2f} | {row['coverage']:.3f} | "
+              f"{row['accuracy']:.3f} | {row['macro_f1']:.3f}")
+
+    # ── §9 action OOD probes ──────────────────────────────────────────────
+    print("\n§9 action OOD")
+    ood_rows = [r for r in json.load(open(os.path.join(OUT_DIR, "ood.json")))]
+    emb_by_idx = {i: np.asarray(e["embedding"], dtype=np.float64)
+                  for i, e in enumerate(slice18_sparse.filter_dev_pool(
+                      slice18_sparse.load_data()[1]))}
+    Xo = np.vstack([emb_by_idx[r["idx"]] for r in ood_rows])
+    fold_models = []
+    for te_fold in sorted(set(folds_arr.tolist())):
+        tr = folds_arr != te_fold
+        clf = slice18_sparse.LogisticRegression(
+            C=bestC, max_iter=2000, solver="lbfgs", random_state=42)
+        clf.fit(X[tr], yt[tr])
+        fold_models.append(clf)
+    # XX
+    # comparable to the non-action in-fold behaviour
+    po = np.zeros((len(Xo), 5))
+    for clf in fold_models:
+        po += clf.predict_proba(Xo)
+    po /= len(fold_models)
+    ood_top = int(np.argmax(po.mean(0)))
+    ood_conf = po.max(1)
+    ood_pred = po.argmax(1)
+    top_dist = {CLASSES[i]: int((ood_pred == i).sum()) for i in range(5)}
+    confident_na = int((ood_conf > 0.9).sum())
+    report["ood"] = {
+        "n": len(ood_rows),
+        "top_class": CLASSES[int(ood_top)],
+        "top_class_dist": top_dist,
+        "conf_gt_0.9": confident_na,
+        "conf_gt_0.9_frac": float(confident_na / len(ood_rows)),
+        "conf_mean": float(ood_conf.mean()),
+        "conf_median": float(np.median(ood_conf)),
+    }
+    print(f"   action OOD n={len(ood_rows)}: most-confident class={report['ood']['top_class']} "
+          f"dist={top_dist}")
+    print(f"   conf>0.9: {confident_na} ({confident_na/len(ood_rows):.3f}) "
+          f"conf mean={report['ood']['conf_mean']:.3f}")
+
+    # ── §10 artifact cost ────────────────────────────────────────────────
+    print("\n§10 artifact")
+    n_params = len(CLASSES) * X.shape[1] + len(CLASSES)
+    fp32 = n_params * 4
+    report["artifact"] = {
+        "e5_dim": X.shape[1],
+        "head_params": n_params,
+        "head_fp32_bytes": fp32,
+        "head_fp32_kib": fp32 / 1024,
+        "head_int8_bytes": n_params,
+    }
+    # incremental latency of the linear head over a batch of 1 (µs)
+    clf = slice18_sparse.LogisticRegression(C=bestC, max_iter=2000, solver="lbfgs", random_state=42)
+    clf.fit(X, yt)
+    x1 = X[:1]
+    for _ in range(50):
+        clf.predict_proba(x1)
+    lat = []
+    for _ in range(2000):
+        t0 = time.perf_counter_ns()
+        clf.predict_proba(x1)
+        lat.append((time.perf_counter_ns() - t0) / 1e3)
+    lat = np.array(lat)
+    report["artifact"]["head_latency_us_mean"] = float(lat.mean())
+    report["artifact"]["head_latency_us_p50"] = float(np.median(lat))
+    print(f"   head params={n_params} fp32={fp32/1024:.2f}KiB "
+          f"lat mean={lat.mean():.2f}us p50={np.median(lat):.2f}us")
+
+    with open(os.path.join(OUT_DIR, "results.json"), "w") as f:
+        json.dump(report, f, ensure_ascii=False, indent=1, default=float)
+    print(f"\nwrote {OUT_DIR}/results.json")
+
+
+if __name__ == "__main__":
+    main()