Maven/cmd/semantic-router-experiment/slice21/main.go

// Slice 21 runner: report the deterministic execution-frame guard against the
// frozen slice-20 dev pool.
//
// Reads the emit step's compact files (pool.json, pairs.json, sparse_oof.json
// in /tmp/mvn-s21) and prints the report tables plus a machine-readable
// guard_results.json. Reuses router/morph/lexicon parsers live inside this
// module — the pool texts are the only data, no embedding is recomputed.
//
// Usage: go run ./cmd/semantic-router-experiment/slice21
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"regexp"
	"sort"
	"strings"
	"unicode"
)

const sparseThreshold = 0.715 // slice-18 §4 strict operating point (P>=0.95 best recall)

var familyPriority = []string{
	"capability_question", "question", "first_person_request",
	"modal_request", "polite_request", "reordered_target", "direct_imperative",
}

func familyOf(tags []string) string {
	for _, f := range familyPriority {
		if hasTok(tags, f) {
			return f
		}
	}
	return "other"
}

// ── pool row ──────────────────────────────────────────────────────────────

type Row struct {
	Idx       int      `json:"idx"`
	Text      string   `json:"text"`
	NText     string   `json:"n_text"`
	Route     string   `json:"route"`
	Y         int      `json:"y"`
	Tags      []string `json:"tags"`
	CVFold    int      `json:"cv_fold"`
	SplitGp   string   `json:"split_group"`
	SourceID  string   `json:"source_id"`
	Family    string
	VariantOf int
}

type Pair struct{ Cap, Act int }

// ── stress variants ───────────────────────────────────────────────────────

var nofinalRe = regexp.MustCompile(`[?.!,;:]+$`)

// strip_punct mirrors the slice-18/19 python strip_punct: trailing sentence
// punctuation, then every non-word/non-space rune.
func stripPunct(t string) string {
	t = nofinalRe.ReplaceAllString(strings.TrimSpace(t), "")
	out := make([]rune, 0, len(t))
	var prevSpace bool
	for _, r := range t {
		if unicode.IsLetter(r) || unicode.IsNumber(r) {
			out = append(out, r)
			prevSpace = false
		} else if !prevSpace {
			out = append(out, ' ')
			prevSpace = true
		}
	}
	return strings.TrimSpace(string(out))
}

func variants(nText string) [3]string {
	return [3]string{
		nText,
		nofinalRe.ReplaceAllString(strings.TrimSpace(nText), ""),
		stripPunct(nText),
	}
}

const (
	vOrig = iota
	vNofinal
	vStrip
)

var variantName = [3]string{"orig", "nofinal", "strip"}

type result struct {
	Frame Frame `json:"frame"`
}

// ── metrics ──────────────────────────────────────────────────────────────

type triTab struct {
	Permissive, Blocked, Ambiguous       int
	PermNonact, BlockedAction, AmbAction int
}

type runAgg struct {
	n, action, tp, fp, fn int
	approved              int
	capPermissive         int
}

func (a *runAgg) addApproved(approved bool, route string) {
	a.n++
	if route == "action" {
		a.action++
	}
	if approved {
		a.approved++
		if route == "action" {
			a.tp++
		} else {
			a.fp++
		}
	} else if route == "action" {
		a.fn++
	}
}

func (a *runAgg) P() string { return fmtPct(frac(a.tp, a.tp+a.fp)) }
func (a *runAgg) R() string { return fmtPct(frac(a.tp, a.action)) }
func (a *runAgg) FA() int   { return a.fp }
func (a *runAgg) FArate() string {
	return fmtPct(frac(a.fp, a.n))
}

func maxi(a, b int) int {
	if a > b {
		return a
	}
	return b
}

// frac is the guarded ratio the tables print (0/0 is 0).
func frac(num, den int) float64 { return float64(num) / float64(maxi(den, 1)) }

func fmtPct(v float64) string { return fmt.Sprintf("%.1f%%", 100*v) }

// ── main ─────────────────────────────────────────────────────────────────

func main() {
	poolPath := flag.String("pool", "/tmp/mvn-s21/pool.json", "dev pool rows")
	pairsPath := flag.String("pairs", "/tmp/mvn-s21/pairs.json", "cap-vs-action pairs")
	sparsePath := flag.String("sparse", "/tmp/mvn-s21/sparse_oof.json", "slice-18 both OOF proba")
	outPath := flag.String("out", "/tmp/mvn-s21/guard_results.json", "machine-readable results")
	flag.Parse()

	rows := mustLoad[[]Row](*poolPath)
	// pairs.json is bare [cap, act] index pairs; adapt into typed pairs.
	rawPairs := mustLoad[[][2]int](*pairsPath)
	pairs := make([]Pair, 0, len(rawPairs))
	for _, rp := range rawPairs {
		pairs = append(pairs, Pair{Cap: rp[0], Act: rp[1]})
	}
	sparseOOF := mustLoad[[]struct {
		Idx   int     `json:"idx"`
		Proba float64 `json:"proba"`
	}](*sparsePath)
	proba := make([]float64, len(rows))
	for _, s := range sparseOOF {
		proba[s.Idx] = s.Proba
	}

	famPrio := 0
	for i := range rows {
		rows[i].Family = familyOf(rows[i].Tags)
		if rows[i].Family != "other" {
			famPrio++
		}
	}
	_ = famPrio

	// verdicts per variant
	type rowRes struct {
		Idx    int               `json:"idx"`
		Text   string            `json:"text"`
		Route  string            `json:"route"`
		Family string            `json:"family"`
		Tags   []string          `json:"tags"`
		Frames map[string]string `json:"frames"` // variant -> eligibility
	}

	perVariant := make([][3]Frame, len(rows))
	fmt.Println("slice 21 — deterministic execution-frame guard on slice-20 dev pool")
	fmt.Println("==================================================================")

	for i, r := range rows {
		vs := variants(r.NText)
		var fr [3]Frame
		for vi := 0; vi < 3; vi++ {
			fr[vi] = Evaluate(vs[vi])
		}
		perVariant[i] = fr
	}

	// ── §3 three-way cross-tab (orig) ─────────────────────────────────────
	fmt.Println("\n## 1. Three-way eligibility × route (orig)")
	tab := triTab{}
	for i, r := range rows {
		switch perVariant[i][vOrig].Eligibility {
		case Permissive:
			tab.Permissive++
			if r.Route != "action" {
				tab.PermNonact++
			}
		case Blocked:
			tab.Blocked++
			if r.Route == "action" {
				tab.BlockedAction++
			}
		case Ambiguous:
			tab.Ambiguous++
			if r.Route == "action" {
				tab.AmbAction++
			}
		}
	}
	fmt.Printf("permissive: %d   blocked: %d   ambiguous: %d\n", tab.Permissive, tab.Blocked, tab.Ambiguous)
	fmt.Printf("  permissive non-action: %d   blocked action: %d   ambiguous action: %d\n",
		tab.PermNonact, tab.BlockedAction, tab.AmbAction)

	// ── §4 binary executable-gate metrics on orig ─────────────────────────
	fmt.Println("\n## 2. Binary executable gate (approve = permissive; deny = blocked|ambiguous)")
	a := runAgg{}
	capCov, capPerm, capAmb := 0, 0, 0
	for i, r := range rows {
		el := perVariant[i][vOrig].Eligibility
		a.addApproved(el == Permissive, r.Route)
		if hasTok(r.Tags, "capability_question") {
			capCov++
			switch el {
			case Permissive:
				capPerm++
			case Ambiguous:
				capAmb++
			}
		}
	}
	fmt.Printf("approved: %d  denied: %d  (n=%d, action=%d)\n", a.approved, a.n-a.approved, a.n, a.action)
	fmt.Printf("action precision %s  recall %s  FA %d (%s)\n", a.P(), a.R(), a.FA(), a.FArate())
	fmt.Printf("capability-question dangerous pass: %d / %d (rate %s)\n",
		capPerm, capCov, fmtPct(frac(capPerm, capCov)))
	fmt.Printf("capability-question blocked %d, ambiguous %d\n", capCov-capPerm-capAmb, capAmb)

	// ── §15 family stress (orig) ──────────────────────────────────────────
	fmt.Println("\n## 3. Family stress (orig; counts per eligibility)")
	fmt.Printf("%-24s %8s %8s %8s %8s\n", "family", "n", "perm", "block", "ambig")
	famOrder := []string{"direct_imperative", "polite_request", "modal_request", "first_person_request",
		"reordered_target", "capability_question", "question", "other"}
	famAgg := map[string]*triTab{}
	for _, f := range famOrder {
		famAgg[f] = &triTab{}
	}
	for i, r := range rows {
		t := famAgg[r.Family]
		if t == nil {
			continue
		}
		switch perVariant[i][vOrig].Eligibility {
		case Permissive:
			t.Permissive++
			if r.Route != "action" {
				t.PermNonact++
			}
		case Blocked:
			t.Blocked++
		case Ambiguous:
			t.Ambiguous++
		}
	}
	for _, f := range famOrder {
		t := famAgg[f]
		if t == nil {
			continue
		}
		n := t.Permissive + t.Blocked + t.Ambiguous
		if n == 0 {
			continue
		}
		fmt.Printf("%-24s %8d %8d %8d %8d\n", f, n, t.Permissive, t.Blocked, t.Ambiguous)
	}

	// ── §cap-Q LOFO across stress variants ────────────────────────────────
	fmt.Println("\n## 4. Capability-question dangerous pass by stress variant")
	for vi := 0; vi < 3; vi++ {
		cp, cb, ca := 0, 0, 0
		for i, r := range rows {
			if !hasTok(r.Tags, "capability_question") {
				continue
			}
			switch perVariant[i][vi].Eligibility {
			case Permissive:
				cp++
			case Blocked:
				cb++
			case Ambiguous:
				ca++
			}
		}
		fmt.Printf("  %-8s dangerous-pass %d  blocked %d  ambiguous %d\n",
			variantName[vi], cp, cb, ca)
	}

	// ── §pair test ────────────────────────────────────────────────────────
	fmt.Println("\n## 5. Paired action/capability (cap row must never clear)")
	capClear, actPerm, actAmbig, actBlock := 0, 0, 0, 0
	for _, p := range pairs {
		cel := perVariant[p.Cap][vOrig].Eligibility
		ael := perVariant[p.Act][vOrig].Eligibility
		if cel == Permissive {
			capClear++
		}
		switch ael {
		case Permissive:
			actPerm++
		case Ambiguous:
			actAmbig++
		case Blocked:
			actBlock++
		}
	}
	fmt.Printf("pairs %d: cap cleared %d (rate %s), action permissive %d, action ambiguous %d, action blocked %d\n",
		len(pairs), capClear, fmtPct(frac(capClear, len(pairs))),
		actPerm, actAmbig, actBlock)

	// ── �safe composition §17 ─────────────────────────────────────────────
	fmt.Println("\n## 6. Composition: guard-alone / sparse-alone / guard→sparse (orig)")
	compose := map[string]*runAgg{
		"guard_alone":  {},
		"sparse_alone": {},
		"guard_sparse": {},
	}
	for i, r := range rows {
		gPerm := perVariant[i][vOrig].Eligibility == Permissive
		sPerm := proba[i] >= sparseThreshold
		compose["guard_alone"].addApproved(gPerm, r.Route)
		compose["sparse_alone"].addApproved(sPerm, r.Route)
		compose["guard_sparse"].addApproved(gPerm && sPerm, r.Route)
	}
	fmt.Printf("%-14s %8s %8s %6s %10s %6s %10s\n", "policy", "P", "R", "FA", "FA rate", "capQ", "capQ rate")
	for _, name := range []string{"guard_alone", "sparse_alone", "guard_sparse"} {
		agg := compose[name]
		capQ := 0
		for i, r := range rows {
			if !hasTok(r.Tags, "capability_question") {
				continue
			}
			ok := false
			switch name {
			case "guard_alone":
				ok = perVariant[i][vOrig].Eligibility == Permissive
			case "sparse_alone":
				ok = proba[i] >= sparseThreshold
			case "guard_sparse":
				ok = perVariant[i][vOrig].Eligibility == Permissive && proba[i] >= sparseThreshold
			}
			if ok {
				capQ++
			}
		}
		fmt.Printf("%-14s %8s %8s %6d %10s %6d %10s\n", name, agg.P(), agg.R(), agg.FA(),
			agg.FArate(), capQ, fmtPct(float64(capQ)/126))
	}

	// composition on strip too (brief §16 voice stress)
	fmt.Println("\n## 7. Composition on punctuation-stripped text (strip)")
	c2 := runAgg{}
	capQ2 := 0
	for i, r := range rows {
		gPerm := perVariant[i][vStrip].Eligibility == Permissive
		ok := gPerm && proba[i] >= sparseThreshold
		c2.addApproved(ok, r.Route)
		if hasTok(r.Tags, "capability_question") && ok {
			capQ2++
		}
	}
	fmt.Printf("guard→sparse strip: P %s  R %s  FA %d (%s)  capQ pass %d\n",
		c2.P(), c2.R(), c2.FA(), c2.FArate(), capQ2)

	// ── §19 manual classification scratch ─────────────────────────────────
	fmt.Println("\n## 8. Manual classification (scan material written to manual_class.json)")
	var dangerous []map[string]any
	var permNonact []map[string]any
	var deniedAction []map[string]any
	for i, r := range rows {
		fr := perVariant[i][vOrig]
		if hasTok(r.Tags, "capability_question") && fr.Eligibility == Permissive {
			dangerous = append(dangerous, map[string]any{
				"idx": r.Idx, "text": r.Text, "route": r.Route,
				"reasons": fr.Reasons,
			})
		}
		if fr.Eligibility == Permissive && r.Route != "action" {
			permNonact = append(permNonact, map[string]any{
				"idx": r.Idx, "text": r.Text, "route": r.Route,
				"family": r.Family, "reasons": fr.Reasons,
			})
		}
		if fr.Eligibility != Permissive && r.Route == "action" {
			deniedAction = append(deniedAction, map[string]any{
				"idx": r.Idx, "text": r.Text, "family": r.Family,
				"eligibility": fr.Eligibility.String(), "reasons": fr.Reasons,
			})
		}
	}
	writeManual(permNonact, deniedAction, dangerous)
	fmt.Printf("dangerous passes: %d   permissive non-action: %d   denied action: %d\n",
		len(dangerous), len(permNonact), len(deniedAction))
	groupAndSample("permissive non-action by reason+family", permNonact, 4)
	groupAndSample("denied action by reason+family", deniedAction, 4)

	// ── fixtures ──────────────────────────────────────────────────────────
	fmt.Println("\n## 9. Brief fixtures")
	pass := 0
	for _, fx := range Fixtures {
		got := Evaluate(fx.Utterance)
		mark := "ok "
		if got.Eligibility != fx.Want {
			mark = "FAIL"
		} else {
			pass++
		}
		if got.Eligibility != fx.Want {
			fmt.Printf("  %s %-14s want %-10s got %-10s  %s\n", mark, fx.Family,
				fx.Want, got.Eligibility.String(), fx.Utterance)
		}
	}
	fmt.Printf("fixtures: %d/%d passed\n", pass, len(Fixtures))

	// write result file
	rr := make([]rowRes, 0, len(rows))
	for i, r := range rows {
		fr := [3]string{"", "", ""}
		for vi := 0; vi < 3; vi++ {
			fr[vi] = perVariant[i][vi].Eligibility.String()
		}
		rr = append(rr, rowRes{
			Idx: r.Idx, Text: r.Text, Route: r.Route, Family: r.Family, Tags: r.Tags,
			Frames: map[string]string{
				"orig": fr[vOrig], "nofinal": fr[vNofinal], "strip": fr[vStrip],
			},
		})
	}
	if *outPath != "" {
		mustSave(*outPath, map[string]any{
			"pool": "/tmp/mvn-s21/pool.json",
			"rows": rr,
			"aggregates": map[string]any{
				"tab":               tab,
				"capq_pass":         capPerm,
				"capq_blocked":      capCov - capPerm - capAmb,
				"capq_ambiguous":    capAmb,
				"binary":            map[string]any{"tp": a.tp, "fp": a.fp, "fn": a.fn, "approved": a.approved, "n": a.n},
				"pairs":             map[string]any{"n": len(pairs), "cap_cleared": capClear, "act_permissive": actPerm},
				"guard_sparse":      map[string]any{"tp": compose["guard_sparse"].tp, "fp": compose["guard_sparse"].fp, "fn": compose["guard_sparse"].fn},
				"dangerous_passes":  len(dangerous),
				"perm_nonact_count": len(permNonact),
				"denied_action":     len(deniedAction),
			},
		})
		fmt.Println("wrote", *outPath)
	}
}

// ── manual classification helpers ─────────────────────────────────────────

func writeManual(permNonact, deniedAction, dangerous []map[string]any) {
	writeJSON("/tmp/mvn-s21/manual_class.json", map[string]any{
		"dangerous_passes":      dangerous,
		"permissive_non_action": permNonact,
		"denied_action":         deniedAction,
	})
}

func groupAndSample(title string, rows []map[string]any, sample int) {
	type g struct {
		key   string
		n     int
		texts []string
	}
	groups := map[string]*g{}
	var order []string
	for _, r := range rows {
		var family, reason, el string
		if v, ok := r["family"].(string); ok {
			family = v
		}
		if v, ok := r["eligibility"].(string); ok {
			el = v
		}
		if rs, ok := r["reasons"].([]Reason); ok {
			rs2 := make([]string, len(rs))
			for k, rr := range rs {
				rs2[k] = rr.String()
			}
			reason = strings.Join(rs2, ",")
		} else if rs, ok := r["reasons"].([]string); ok {
			reason = strings.Join(rs, ",")
		}
		key := fmt.Sprintf("family=%s elig=%s reason=%s", family, el, reason)
		if _, ok := groups[key]; !ok {
			groups[key] = &g{key: key}
			order = append(order, key)
		}
		groups[key].n++
		if len(groups[key].texts) < sample {
			groups[key].texts = append(groups[key].texts, firstN(fmt.Sprint(r["text"]), 60))
		}
	}
	fmt.Printf("%s (%d rows):\n", title, len(rows))
	for _, key := range order {
		gr := groups[key]
		fmt.Printf("  %-58s n=%d  %s\n", gr.key, gr.n, strings.Join(gr.texts, " | "))
	}
}

func firstN(s string, n int) string {
	if len(s) <= n {
		return s
	}
	return s[:n] + "…"
}

// ── io helpers ────────────────────────────────────────────────────────────

func mustLoad[T any](path string) T {
	b, err := os.ReadFile(path)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	var v T
	if err := json.Unmarshal(b, &v); err != nil {
		fmt.Fprintln(os.Stderr, "json:", err)
		os.Exit(1)
	}
	return v
}

func mustSave(path string, v any) {
	b, err := json.MarshalIndent(v, "", "  ")
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	if err := os.WriteFile(path, b, 0o644); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func writeJSON(path string, v any) {
	b, _ := json.MarshalIndent(v, "", "  ")
	_ = os.WriteFile(path, b, 0o644)
}

var _ = sort.Strings