router/semantic: slice 21 deterministic execution-frame guard engine, fixtures, runner and emit step

The guard answers one question — may this utterance become an executable
action — as a three-way policy gate (permissive / blocked / ambiguous) and
never decides what the utterance is. Rules are the encoding of the measured
slice-20 dev-pool discriminators: 126 capability-question rows are 42/42/42
addressed / bare-ability / bare-future; 127 bare можешь+пожалуйста rows are
100% action; can-you-please is 100% action. Reuses the shipped prohibition
parser, morph finiteness and lexicon fillers; reason vocabulary is closed.

Slice 21 (task/725, brief after the accepted slice 20).
This commit is contained in:
2026-09-07 23:43:38 +04:00
parent de1cb40456
commit fa98e4722e
5 changed files with 1594 additions and 0 deletions
@@ -0,0 +1,566 @@
// Slice 21 runner: report the deterministic execution-frame guard against the
// frozen slice-20 dev pool.
//
// Reads the emit step's compact files (pool.json, pairs.json, sparse_oof.json
// in /tmp/mvn-s21) and prints the report tables plus a machine-readable
// guard_results.json. Reuses router/morph/lexicon parsers live inside this
// module — the pool texts are the only data, no embedding is recomputed.
//
// Usage: go run ./cmd/semantic-router-experiment/slice21
package main
import (
"encoding/json"
"flag"
"fmt"
"os"
"regexp"
"sort"
"strings"
"unicode"
)
const sparseThreshold = 0.715 // slice-18 §4 strict operating point (P>=0.95 best recall)
var familyPriority = []string{
"capability_question", "question", "first_person_request",
"modal_request", "polite_request", "reordered_target", "direct_imperative",
}
func familyOf(tags []string) string {
for _, f := range familyPriority {
if hasTok(tags, f) {
return f
}
}
return "other"
}
// ── pool row ──────────────────────────────────────────────────────────────
type Row struct {
Idx int `json:"idx"`
Text string `json:"text"`
NText string `json:"n_text"`
Route string `json:"route"`
Y int `json:"y"`
Tags []string `json:"tags"`
CVFold int `json:"cv_fold"`
SplitGp string `json:"split_group"`
SourceID string `json:"source_id"`
Family string
VariantOf int
}
type Pair struct{ Cap, Act int }
// ── stress variants ───────────────────────────────────────────────────────
var nofinalRe = regexp.MustCompile(`[?.!,;:]+$`)
// strip_punct mirrors the slice-18/19 python strip_punct: trailing sentence
// punctuation, then every non-word/non-space rune.
func stripPunct(t string) string {
t = nofinalRe.ReplaceAllString(strings.TrimSpace(t), "")
out := make([]rune, 0, len(t))
var prevSpace bool
for _, r := range t {
if unicode.IsLetter(r) || unicode.IsNumber(r) {
out = append(out, r)
prevSpace = false
} else if !prevSpace {
out = append(out, ' ')
prevSpace = true
}
}
return strings.TrimSpace(string(out))
}
func variants(nText string) [3]string {
return [3]string{
nText,
nofinalRe.ReplaceAllString(strings.TrimSpace(nText), ""),
stripPunct(nText),
}
}
const (
vOrig = iota
vNofinal
vStrip
)
var variantName = [3]string{"orig", "nofinal", "strip"}
type result struct {
Frame Frame `json:"frame"`
}
// ── metrics ──────────────────────────────────────────────────────────────
type triTab struct {
Permissive, Blocked, Ambiguous int
PermNonact, BlockedAction, AmbAction int
}
type runAgg struct {
n, action, tp, fp, fn int
approved int
capPermissive int
}
func (a *runAgg) addApproved(approved bool, route string) {
a.n++
if route == "action" {
a.action++
}
if approved {
a.approved++
if route == "action" {
a.tp++
} else {
a.fp++
}
} else if route == "action" {
a.fn++
}
}
func (a *runAgg) P() string { return fmtPct(frac(a.tp, a.tp+a.fp)) }
func (a *runAgg) R() string { return fmtPct(frac(a.tp, a.action)) }
func (a *runAgg) FA() int { return a.fp }
func (a *runAgg) FArate() string {
return fmtPct(frac(a.fp, a.n))
}
func maxi(a, b int) int {
if a > b {
return a
}
return b
}
// frac is the guarded ratio the tables print (0/0 is 0).
func frac(num, den int) float64 { return float64(num) / float64(maxi(den, 1)) }
func fmtPct(v float64) string { return fmt.Sprintf("%.1f%%", 100*v) }
// ── main ─────────────────────────────────────────────────────────────────
func main() {
poolPath := flag.String("pool", "/tmp/mvn-s21/pool.json", "dev pool rows")
pairsPath := flag.String("pairs", "/tmp/mvn-s21/pairs.json", "cap-vs-action pairs")
sparsePath := flag.String("sparse", "/tmp/mvn-s21/sparse_oof.json", "slice-18 both OOF proba")
outPath := flag.String("out", "/tmp/mvn-s21/guard_results.json", "machine-readable results")
flag.Parse()
rows := mustLoad[[]Row](*poolPath)
// pairs.json is bare [cap, act] index pairs; adapt into typed pairs.
rawPairs := mustLoad[[][2]int](*pairsPath)
pairs := make([]Pair, 0, len(rawPairs))
for _, rp := range rawPairs {
pairs = append(pairs, Pair{Cap: rp[0], Act: rp[1]})
}
sparseOOF := mustLoad[[]struct {
Idx int `json:"idx"`
Proba float64 `json:"proba"`
}](*sparsePath)
proba := make([]float64, len(rows))
for _, s := range sparseOOF {
proba[s.Idx] = s.Proba
}
famPrio := 0
for i := range rows {
rows[i].Family = familyOf(rows[i].Tags)
if rows[i].Family != "other" {
famPrio++
}
}
_ = famPrio
// verdicts per variant
type rowRes struct {
Idx int `json:"idx"`
Text string `json:"text"`
Route string `json:"route"`
Family string `json:"family"`
Tags []string `json:"tags"`
Frames map[string]string `json:"frames"` // variant -> eligibility
}
perVariant := make([][3]Frame, len(rows))
fmt.Println("slice 21 — deterministic execution-frame guard on slice-20 dev pool")
fmt.Println("==================================================================")
for i, r := range rows {
vs := variants(r.NText)
var fr [3]Frame
for vi := 0; vi < 3; vi++ {
fr[vi] = Evaluate(vs[vi])
}
perVariant[i] = fr
}
// ── §3 three-way cross-tab (orig) ─────────────────────────────────────
fmt.Println("\n## 1. Three-way eligibility × route (orig)")
tab := triTab{}
for i, r := range rows {
switch perVariant[i][vOrig].Eligibility {
case Permissive:
tab.Permissive++
if r.Route != "action" {
tab.PermNonact++
}
case Blocked:
tab.Blocked++
if r.Route == "action" {
tab.BlockedAction++
}
case Ambiguous:
tab.Ambiguous++
if r.Route == "action" {
tab.AmbAction++
}
}
}
fmt.Printf("permissive: %d blocked: %d ambiguous: %d\n", tab.Permissive, tab.Blocked, tab.Ambiguous)
fmt.Printf(" permissive non-action: %d blocked action: %d ambiguous action: %d\n",
tab.PermNonact, tab.BlockedAction, tab.AmbAction)
// ── §4 binary executable-gate metrics on orig ─────────────────────────
fmt.Println("\n## 2. Binary executable gate (approve = permissive; deny = blocked|ambiguous)")
a := runAgg{}
capCov, capPerm, capAmb := 0, 0, 0
for i, r := range rows {
el := perVariant[i][vOrig].Eligibility
a.addApproved(el == Permissive, r.Route)
if hasTok(r.Tags, "capability_question") {
capCov++
switch el {
case Permissive:
capPerm++
case Ambiguous:
capAmb++
}
}
}
fmt.Printf("approved: %d denied: %d (n=%d, action=%d)\n", a.approved, a.n-a.approved, a.n, a.action)
fmt.Printf("action precision %s recall %s FA %d (%s)\n", a.P(), a.R(), a.FA(), a.FArate())
fmt.Printf("capability-question dangerous pass: %d / %d (rate %s)\n",
capPerm, capCov, fmtPct(frac(capPerm, capCov)))
fmt.Printf("capability-question blocked %d, ambiguous %d\n", capCov-capPerm-capAmb, capAmb)
// ── §15 family stress (orig) ──────────────────────────────────────────
fmt.Println("\n## 3. Family stress (orig; counts per eligibility)")
fmt.Printf("%-24s %8s %8s %8s %8s\n", "family", "n", "perm", "block", "ambig")
famOrder := []string{"direct_imperative", "polite_request", "modal_request", "first_person_request",
"reordered_target", "capability_question", "question", "other"}
famAgg := map[string]*triTab{}
for _, f := range famOrder {
famAgg[f] = &triTab{}
}
for i, r := range rows {
t := famAgg[r.Family]
if t == nil {
continue
}
switch perVariant[i][vOrig].Eligibility {
case Permissive:
t.Permissive++
if r.Route != "action" {
t.PermNonact++
}
case Blocked:
t.Blocked++
case Ambiguous:
t.Ambiguous++
}
}
for _, f := range famOrder {
t := famAgg[f]
if t == nil {
continue
}
n := t.Permissive + t.Blocked + t.Ambiguous
if n == 0 {
continue
}
fmt.Printf("%-24s %8d %8d %8d %8d\n", f, n, t.Permissive, t.Blocked, t.Ambiguous)
}
// ── §cap-Q LOFO across stress variants ────────────────────────────────
fmt.Println("\n## 4. Capability-question dangerous pass by stress variant")
for vi := 0; vi < 3; vi++ {
cp, cb, ca := 0, 0, 0
for i, r := range rows {
if !hasTok(r.Tags, "capability_question") {
continue
}
switch perVariant[i][vi].Eligibility {
case Permissive:
cp++
case Blocked:
cb++
case Ambiguous:
ca++
}
}
fmt.Printf(" %-8s dangerous-pass %d blocked %d ambiguous %d\n",
variantName[vi], cp, cb, ca)
}
// ── §pair test ────────────────────────────────────────────────────────
fmt.Println("\n## 5. Paired action/capability (cap row must never clear)")
capClear, actPerm, actAmbig, actBlock := 0, 0, 0, 0
for _, p := range pairs {
cel := perVariant[p.Cap][vOrig].Eligibility
ael := perVariant[p.Act][vOrig].Eligibility
if cel == Permissive {
capClear++
}
switch ael {
case Permissive:
actPerm++
case Ambiguous:
actAmbig++
case Blocked:
actBlock++
}
}
fmt.Printf("pairs %d: cap cleared %d (rate %s), action permissive %d, action ambiguous %d, action blocked %d\n",
len(pairs), capClear, fmtPct(frac(capClear, len(pairs))),
actPerm, actAmbig, actBlock)
// ── safe composition §17 ─────────────────────────────────────────────
fmt.Println("\n## 6. Composition: guard-alone / sparse-alone / guard→sparse (orig)")
compose := map[string]*runAgg{
"guard_alone": {},
"sparse_alone": {},
"guard_sparse": {},
}
for i, r := range rows {
gPerm := perVariant[i][vOrig].Eligibility == Permissive
sPerm := proba[i] >= sparseThreshold
compose["guard_alone"].addApproved(gPerm, r.Route)
compose["sparse_alone"].addApproved(sPerm, r.Route)
compose["guard_sparse"].addApproved(gPerm && sPerm, r.Route)
}
fmt.Printf("%-14s %8s %8s %6s %10s %6s %10s\n", "policy", "P", "R", "FA", "FA rate", "capQ", "capQ rate")
for _, name := range []string{"guard_alone", "sparse_alone", "guard_sparse"} {
agg := compose[name]
capQ := 0
for i, r := range rows {
if !hasTok(r.Tags, "capability_question") {
continue
}
ok := false
switch name {
case "guard_alone":
ok = perVariant[i][vOrig].Eligibility == Permissive
case "sparse_alone":
ok = proba[i] >= sparseThreshold
case "guard_sparse":
ok = perVariant[i][vOrig].Eligibility == Permissive && proba[i] >= sparseThreshold
}
if ok {
capQ++
}
}
fmt.Printf("%-14s %8s %8s %6d %10s %6d %10s\n", name, agg.P(), agg.R(), agg.FA(),
agg.FArate(), capQ, fmtPct(float64(capQ)/126))
}
// composition on strip too (brief §16 voice stress)
fmt.Println("\n## 7. Composition on punctuation-stripped text (strip)")
c2 := runAgg{}
capQ2 := 0
for i, r := range rows {
gPerm := perVariant[i][vStrip].Eligibility == Permissive
ok := gPerm && proba[i] >= sparseThreshold
c2.addApproved(ok, r.Route)
if hasTok(r.Tags, "capability_question") && ok {
capQ2++
}
}
fmt.Printf("guard→sparse strip: P %s R %s FA %d (%s) capQ pass %d\n",
c2.P(), c2.R(), c2.FA(), c2.FArate(), capQ2)
// ── §19 manual classification scratch ─────────────────────────────────
fmt.Println("\n## 8. Manual classification (scan material written to manual_class.json)")
var dangerous []map[string]any
var permNonact []map[string]any
var deniedAction []map[string]any
for i, r := range rows {
fr := perVariant[i][vOrig]
if hasTok(r.Tags, "capability_question") && fr.Eligibility == Permissive {
dangerous = append(dangerous, map[string]any{
"idx": r.Idx, "text": r.Text, "route": r.Route,
"reasons": fr.Reasons,
})
}
if fr.Eligibility == Permissive && r.Route != "action" {
permNonact = append(permNonact, map[string]any{
"idx": r.Idx, "text": r.Text, "route": r.Route,
"family": r.Family, "reasons": fr.Reasons,
})
}
if fr.Eligibility != Permissive && r.Route == "action" {
deniedAction = append(deniedAction, map[string]any{
"idx": r.Idx, "text": r.Text, "family": r.Family,
"eligibility": fr.Eligibility.String(), "reasons": fr.Reasons,
})
}
}
writeManual(permNonact, deniedAction, dangerous)
fmt.Printf("dangerous passes: %d permissive non-action: %d denied action: %d\n",
len(dangerous), len(permNonact), len(deniedAction))
groupAndSample("permissive non-action by reason+family", permNonact, 4)
groupAndSample("denied action by reason+family", deniedAction, 4)
// ── fixtures ──────────────────────────────────────────────────────────
fmt.Println("\n## 9. Brief fixtures")
pass := 0
for _, fx := range Fixtures {
got := Evaluate(fx.Utterance)
mark := "ok "
if got.Eligibility != fx.Want {
mark = "FAIL"
} else {
pass++
}
if got.Eligibility != fx.Want {
fmt.Printf(" %s %-14s want %-10s got %-10s %s\n", mark, fx.Family,
fx.Want, got.Eligibility.String(), fx.Utterance)
}
}
fmt.Printf("fixtures: %d/%d passed\n", pass, len(Fixtures))
// write result file
rr := make([]rowRes, 0, len(rows))
for i, r := range rows {
fr := [3]string{"", "", ""}
for vi := 0; vi < 3; vi++ {
fr[vi] = perVariant[i][vi].Eligibility.String()
}
rr = append(rr, rowRes{
Idx: r.Idx, Text: r.Text, Route: r.Route, Family: r.Family, Tags: r.Tags,
Frames: map[string]string{
"orig": fr[vOrig], "nofinal": fr[vNofinal], "strip": fr[vStrip],
},
})
}
if *outPath != "" {
mustSave(*outPath, map[string]any{
"pool": "/tmp/mvn-s21/pool.json",
"rows": rr,
"aggregates": map[string]any{
"tab": tab,
"capq_pass": capPerm,
"capq_blocked": capCov - capPerm - capAmb,
"capq_ambiguous": capAmb,
"binary": map[string]any{"tp": a.tp, "fp": a.fp, "fn": a.fn, "approved": a.approved, "n": a.n},
"pairs": map[string]any{"n": len(pairs), "cap_cleared": capClear, "act_permissive": actPerm},
"guard_sparse": map[string]any{"tp": compose["guard_sparse"].tp, "fp": compose["guard_sparse"].fp, "fn": compose["guard_sparse"].fn},
"dangerous_passes": len(dangerous),
"perm_nonact_count": len(permNonact),
"denied_action": len(deniedAction),
},
})
fmt.Println("wrote", *outPath)
}
}
// ── manual classification helpers ─────────────────────────────────────────
func writeManual(permNonact, deniedAction, dangerous []map[string]any) {
writeJSON("/tmp/mvn-s21/manual_class.json", map[string]any{
"dangerous_passes": dangerous,
"permissive_non_action": permNonact,
"denied_action": deniedAction,
})
}
func groupAndSample(title string, rows []map[string]any, sample int) {
type g struct {
key string
n int
texts []string
}
groups := map[string]*g{}
var order []string
for _, r := range rows {
var family, reason, el string
if v, ok := r["family"].(string); ok {
family = v
}
if v, ok := r["eligibility"].(string); ok {
el = v
}
if rs, ok := r["reasons"].([]Reason); ok {
rs2 := make([]string, len(rs))
for k, rr := range rs {
rs2[k] = rr.String()
}
reason = strings.Join(rs2, ",")
} else if rs, ok := r["reasons"].([]string); ok {
reason = strings.Join(rs, ",")
}
key := fmt.Sprintf("family=%s elig=%s reason=%s", family, el, reason)
if _, ok := groups[key]; !ok {
groups[key] = &g{key: key}
order = append(order, key)
}
groups[key].n++
if len(groups[key].texts) < sample {
groups[key].texts = append(groups[key].texts, firstN(fmt.Sprint(r["text"]), 60))
}
}
fmt.Printf("%s (%d rows):\n", title, len(rows))
for _, key := range order {
gr := groups[key]
fmt.Printf(" %-58s n=%d %s\n", gr.key, gr.n, strings.Join(gr.texts, " | "))
}
}
func firstN(s string, n int) string {
if len(s) <= n {
return s
}
return s[:n] + "…"
}
// ── io helpers ────────────────────────────────────────────────────────────
func mustLoad[T any](path string) T {
b, err := os.ReadFile(path)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
var v T
if err := json.Unmarshal(b, &v); err != nil {
fmt.Fprintln(os.Stderr, "json:", err)
os.Exit(1)
}
return v
}
func mustSave(path string, v any) {
b, err := json.MarshalIndent(v, "", " ")
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if err := os.WriteFile(path, b, 0o644); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func writeJSON(path string, v any) {
b, _ := json.MarshalIndent(v, "", " ")
_ = os.WriteFile(path, b, 0o644)
}
var _ = sort.Strings