From fa98e4722e5992e327be4fe2c183f2dd2d11d39e Mon Sep 17 00:00:00 2001 From: claude Date: Mon, 7 Sep 2026 23:43:38 +0400 Subject: [PATCH] router/semantic: slice 21 deterministic execution-frame guard engine, fixtures, runner and emit step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guard answers one question — may this utterance become an executable action — as a three-way policy gate (permissive / blocked / ambiguous) and never decides what the utterance is. Rules are the encoding of the measured slice-20 dev-pool discriminators: 126 capability-question rows are 42/42/42 addressed / bare-ability / bare-future; 127 bare можешь+пожалуйста rows are 100% action; can-you-please is 100% action. Reuses the shipped prohibition parser, morph finiteness and lexicon fillers; reason vocabulary is closed. Slice 21 (task/725, brief after the accepted slice 20). --- .../slice21/fixtures.go | 95 +++ .../slice21/guard.go | 659 ++++++++++++++++++ .../slice21/guard_test.go | 139 ++++ .../slice21/main.go | 566 +++++++++++++++ .../slice21_emit.py | 135 ++++ 5 files changed, 1594 insertions(+) create mode 100644 cmd/semantic-router-experiment/slice21/fixtures.go create mode 100644 cmd/semantic-router-experiment/slice21/guard.go create mode 100644 cmd/semantic-router-experiment/slice21/guard_test.go create mode 100644 cmd/semantic-router-experiment/slice21/main.go create mode 100644 cmd/semantic-router-experiment/slice21_emit.py diff --git a/cmd/semantic-router-experiment/slice21/fixtures.go b/cmd/semantic-router-experiment/slice21/fixtures.go new file mode 100644 index 0000000..b4d27e8 --- /dev/null +++ b/cmd/semantic-router-experiment/slice21/fixtures.go @@ -0,0 +1,95 @@ +package main + +// Fixture is one brief-specified utterance and the eligibility it must earn. +// Fixtures are EXACTLY the slice-21 brief examples plus the corpus-verified +// structural discriminators. They are eval material, not corpus additions: +// nothing here is inserted into any pool file (brief §2, §18). +type Fixture struct { + Utterance string + Want Eligibility + Family string + Note string +} + +// Fixtures covers every family the brief's §15 list exercises, including the +// four that have no dev-pool rows at all (negation, reported_speech, +// quotation, hypothetical): +// +// direct imperative already plural in dev; fixtures pin the canonical +// cases together with the politeness-negative modal. +// polite request polite imperative + bare "можешь …, пожалуйста". +// modal request "можно …", English "can you …, please". +// first-person request "я хочу …", "мне нужно …", "надо …". +// reordered target "свет выключи, пожалуйста" (target-first command). +// capability question ты/умеешь/сможешь/способна + "… ли" permission. +// ordinary question "что запущено" — no execution pressure. +// negation "не выключай свет" (parser-covered); advisory "не +// надо выключать свет". +// reported speech past/third-person report verbs over a command. +// quotation quoted command, with and without a reporting frame. +// hypothetical если-scopes that are not real condition→command. +var Fixtures = []Fixture{ + // negation — direct commands the prosecutor forbids + {"не выключай свет", Blocked, "negation", "direct prohibition"}, + {"не перезапускай сервер", Blocked, "negation", "direct prohibition"}, + {"не включай nginx", Blocked, "negation", "direct prohibition"}, + {"не надо выключать свет", Blocked, "negation", "advisory negative"}, + {"не стоит перезапускать nginx", Blocked, "negation", "advisory negative"}, + {"не забудь напомнить про свет", Permissive, "negation", "prohibition-parser reminder exemption"}, + + // reported speech — reports an order, does not issue one + {"он сказал выключить свет", Blocked, "reported_speech", "past report verb + infinitive"}, + {"она попросила перезапустить nginx", Blocked, "reported_speech", "past report verb + infinitive"}, + {"мне сказали включить свет", Blocked, "reported_speech", "passive report + infinitive"}, + {"он написал: «перезапусти nginx»", Blocked, "reported_speech", "report verb + quoted imperative"}, + {"скажи мне, что он сказал про свет", Permissive, "reported_speech", "request to report, no commanded clause"}, + {"расскажи про свет", Permissive, "reported_speech", "narrative request, not a reported order"}, + + // quotation — quoted text is referenced, not issued + {"фраза «выключи свет»", Blocked, "quotation", "reporting noun + quoted imperative"}, + {"он сказал «выключи свет»", Blocked, "quotation", "report verb + quoted imperative"}, + {"«выключи свет»", Ambiguous, "quotation", "bare quoted command, no frame"}, + {"выключи свет", Permissive, "quotation", "unquoted imperative is a live command"}, + + // hypothetical + {"если выключить свет...", Blocked, "hypothetical", "conditional + infinitive + ellipsis"}, + {"если бы перезапустить nginx...", Blocked, "hypothetical", "conditional + бы + infinitive"}, + {"что будет если выключить свет", Blocked, "hypothetical", "question-scoped conditional"}, + {"если будет дождь, выключи полив", Permissive, "hypothetical", "real condition → imperative"}, + {"выключи свет если будет дождь", Permissive, "hypothetical", "imperative → real condition"}, + + // capability question — blocked even polite + {"ты можешь выключить свет?", Blocked, "capability_question", "ты + можешь + ?"}, + {"ты можешь выключить свет", Blocked, "capability_question", "ты + можешь, no ?"}, + {"ты можешь выключить свет, пожалуйста", Blocked, "capability_question", "ты + можешь + politeness (42/42 non-action)"}, + {"умеешь ли ты выключить свет", Blocked, "capability_question", "ability form + ли"}, + {"сможешь открыть окно, пожалуйста", Blocked, "capability_question", "bare future + politeness (7/7 non-action)"}, + {"ты способна выключить свет", Blocked, "capability_question", "ты + способна"}, + {"могу ли я выключить свет", Blocked, "capability_question", "first-person can + ли"}, + {"можно ли выключить свет", Blocked, "capability_question", "можно + ли permission question"}, + {"ты выключишь свет?", Ambiguous, "capability_question", "future tense + ? without can-form"}, + + // modal / polite requests — permissive + {"можешь выключить свет, пожалуйста", Permissive, "modal_request", "bare можешь + politeness (127/127 action)"}, + {"пожалуйста, выключи свет", Permissive, "polite_request", "leading politeness + imperative"}, + {"выключи свет, пожалуйста", Permissive, "polite_request", "imperative + trailing politeness"}, + {"выключи свет", Permissive, "direct_imperative", "plain imperative"}, + {"свет выключи, пожалуйста", Permissive, "reordered_target", "target-first imperative"}, + {"can you выключи свет, please", Permissive, "modal_request", "English frame + Russian imperative + please (96/96 action)"}, + {"can you выключи свет", Ambiguous, "modal_request", "English can without politeness"}, + {"не мог бы ты выключить свет", Permissive, "polite_request", "conditional politeness, prohibition-parser exemption"}, + {"можно выключить свет", Permissive, "modal_request", "можно + infinitive permission-implicature request"}, + + // first-person requests + {"я хочу выключить свет", Permissive, "first_person_request", "first-person + illocution"}, + {"я хочу чтобы ты выключил свет", Permissive, "first_person_request", "first-person + embedded ya-you wish"}, + {"надо выключить свет", Permissive, "first_person_request", "impersonal need"}, + {"мне нужно включить свет", Permissive, "first_person_request", "first-person oblique + need"}, + + // ordinary questions — no execution pressure even when answerable + {"что запущено", Ambiguous, "question", "status question, no request evidence"}, + {"какие службы работают", Ambiguous, "question", "question, no request evidence"}, + {"сколько ламп включено", Ambiguous, "question", "question, no request evidence"}, + {"что ты можешь включить", Blocked, "capability_question", "open question with ты + можешь"}, + {"покажи что запущено", Permissive, "first_person_request", "imperative lead over a status question"}, +} diff --git a/cmd/semantic-router-experiment/slice21/guard.go b/cmd/semantic-router-experiment/slice21/guard.go new file mode 100644 index 0000000..023c506 --- /dev/null +++ b/cmd/semantic-router-experiment/slice21/guard.go @@ -0,0 +1,659 @@ +// Guard is the slice-21 deterministic execution-frame engine (experiment-only). +// +// It answers one question: given an utterance, what is its execution-frame +// eligibility as a three-way gate — permissive, blocked, ambiguous — and why. +// It never decides what an utterance IS (that stays with the route classifier); +// it only decides whether an utterance may become an executable action at all. +// The policy is asymmetric on purpose: blocked and ambiguous must never +// execute, and permissive only means "no blocking speech-act evidence exists", +// not "execute this". +// +// It reuses the shipped deterministic routers rather than inventing new ones: +// +// router.ParseCommandProhibition / IsCommandProhibition direct negative commands +// morph.IsVerbForm / morph.Lemma verb mood and finiteness +// lexicon.IsFillerParticle / FirstPerson() politeness and first-person frames +// +// Everything else is closed-class evidence measured on the frozen slice-20 dev +// pool (§"measured discriminators" in the brief): 126 capability-question rows +// split 42/42/42 across ты-addressed, bare ability (умеешь), and bare future +// (сможешь) modality; 127 bare "можешь, пожалуйста" rows are 100% action; +// "can you … , please" (English frame + Russian imperative) is 100% action. +// The rules below are the encoding of precisely those numbers. +// +// The reason vocabulary is a closed set. Additions are design decisions that +// must land in the report, not silent new branches. +package main + +import ( + "strings" + "unicode" + + "github.com/kami/maven/internal/lexicon" + "github.com/kami/maven/internal/morph" + "github.com/kami/maven/internal/router" +) + +// Eligibility is the three-way execution-frame verdict. +type Eligibility int + +const ( + Permissive Eligibility = iota // no blocking speech-act evidence; downstream route decides + Blocked // a speech act forbids execution (negation, question, report, …) + Ambiguous // not enough evidence either way; must not execute +) + +func (e Eligibility) String() string { + switch e { + case Permissive: + return "permissive" + case Blocked: + return "blocked" + default: + return "ambiguous" + } +} + +// Reason is a closed set of structural explanations for a verdict. +type Reason string + +const ( + ReasonCommandProhibition Reason = "command_prohibition" + ReasonCapabilityQuestion Reason = "capability_question" + ReasonReportedSpeech Reason = "reported_speech" + ReasonQuotation Reason = "quotation" + ReasonHypothetical Reason = "hypothetical" + ReasonNegatedCommand Reason = "negated_command" + ReasonExplicitRequest Reason = "explicit_request" + ReasonAmbiguousModal Reason = "ambiguous_modal" + ReasonNoRequestEvidence Reason = "no_request_evidence" +) + +func (r Reason) String() string { return string(r) } + +// Frame is the verdict for one utterance. Eligibility decides; Reasons explain. +// A frame may carry more than one reason (e.g. a quoted reported command). +type Frame struct { + Eligibility Eligibility + Reasons []Reason +} + +// maybeWord is a single-token or multi-token closed expression, e.g. the +// token "не мог бы" covers the three tokens не мog бы when matched as a +// contiguous run ("бы" is itself a bound marker). Multi-token members are +// matched over the reconstructed token text, never over raw text, so +// punctuation boundaries do not defeat them. +type maybeWord struct { + single []string + multi []string // matched as contiguous lowercased token runs +} + +func (w maybeWord) in(toks []string, joined string) bool { + if hasAny(toks, w.single) { + return true + } + for _, m := range w.multi { + tm := strings.Join(tokens(m), " ") + if tm != "" && strings.Contains(joined, tm) { + return true + } + } + return false +} + +// ── closed evidence sets (all measured on the slice-20 dev pool) ────────── + +// wakeAddr is stripped from the left of an utterance before command-form +// detection: "мавен, выключи свет" and "выключи свет" must ride the same +// frame. Closed: the names Maven answers to in the dev pool. +var wakeAddr = []string{"мавен", "maven", "мавэн", "алекса", "алиса", "окей", "эй", "hey"} + +// ruAddress are the second-person Russian address tokens. "ты можешь …" +// (with or without politeness) is 42/42 capability-question in the dev pool, +// so any addressed Russian can-form is a capability question, never a request. +var ruAddress = []string{"ты", "тебе", "тебя", "тобой", "тобою", "вы", "вас", "вам", "вами"} + +// enAddress is the English second-person address. Unlike Russian, "can you … +// , please" is 96/96 action in the dev pool (English modal frame around a +// Russian imperative), so English address alone never blocks: it routes to the +// politeness arm. +var enAddress = []string{"you", "u", "your"} + +// ruCanForms are the present-can verb forms. Bare (no address) "можешь …, +// пожалуйста" is 127/127 action; bare "можешь …" with no politeness is the +// ambiguous bucket (no such rows exist in dev — conservative default). +var ruCanForms = []string{"можешь", "можете", "могу", "можем"} + +// ruAbilityForms are future/ability modal forms that read as a question of +// capability regardless of politeness: "сможешь открыть окно, пожалуйста" and +// "умеешь ли ты …" are 0/84 action in the dev pool, so even a polite bare +// form never grants execution. "мог(ла) бы …" and "смог(ла) бы …" are the +// conditional-politeness mask over the same boundary — except the leading +// politeness construction "не мог бы ты …", which the prohibition parser +// already classifies as ordinary modal politeness and must stay permissive. +var ruAbilityForms = maybeWord{ + single: []string{ + "сможешь", "сможете", "смогу", "сможем", "сумеешь", "сумеете", + "умеешь", "умеете", "способна", "способен", "способно", "способны", + "смог", "смогла", "смогли", "мог", "могла", "могли", + }, + multi: []string{ + "смог бы", "смогла бы", "смогли бы", "мог бы", "могла бы", "могли бы", + "смочь бы", "мочь бы", + }, +} + +// politeNegativeModal is the leading "не мог бы ты/вы …" politeness framing the +// prohibition parser exempts as ordinary modal politeness. When it leads the +// utterance the capability stage declines and the frame reads as a request. +var politeNegativeModal = []string{ + "не мог бы", "не могла бы", "не могли бы", "не смог бы", "не смогла бы", "не смогли бы", +} + +// enCanForms are the English modal can/could tokens. +var enCanForms = []string{"can", "could"} + +// politeness is the closed set of politeness fillers. пожалуйста/плиз/please +// are already closed-class filler particles in the lexicon; the добр-forms +// are the only additions the dev pool exercises. +var politeness = maybeWord{ + single: []string{"пожалуйста", "плиз", "please"}, + multi: []string{"будь добр", "будьте добры", "был бы добр", "были бы добры"}, +} + +// reportVerbs are the past/third-person report verbs — the frame that reports +// a command rather than issuing it. Second-person imperatives ("скажи", +// "расскажи", "напомни") are deliberately absent: those are requests to +// report, and their clause forms part of the current utterance, not a +// replayed order. Matched as closed list (a report verb outside it is a data +// gap, noted in the report). +var reportVerbs = []string{ + "сказал", "сказала", "сказали", "говорил", "говорила", "говорили", + "говорит", "говорят", "попросил", "попросила", "попросили", + "просил", "просила", "просили", "написал", "написала", "написали", + "пишет", "приказал", "приказала", "приказали", "велел", "велела", + "велели", "скомандовал", "скомандовала", "рекомендовал", "рекомендовала", + "посоветовал", "посоветовала", "сообщил", "сообщила", "сообщили", + "объявил", "объявила", "велено", "сказано", "написано", "записано", +} + +// reportNouns name a quoted or reported text: "фраза «выключи свет»" is a +// quotation, not a command. +var reportNouns = []string{ + "фраза", "фразы", "фразе", "фразу", "слово", "слова", "слове", "словом", + "выражение", "выражения", "цитата", "цитату", "цитате", + "название", "текст", "сообщение", "письмо", "заметка", "заметку", +} + +// hypothesisMarkers open a conditional scope. +var hypothesisMarkers = []string{"если", "ежели", "коли", "кабы", "if"} + +// illocutionVerbs make a first-person or impersonal clause a request even +// without an imperative form ("я хочу …", "мне нужно …", "надо …"). +var illocutionVerbs = maybeWord{ + single: []string{ + "хочу", "хотел", "хотела", "хотелось", "желаю", "прошу", "просим", + "просил", "просила", "просили", "попросить", + "надо", "нужно", "следует", "пора", "требуется", "придётся", "придется", + "могу", "давай", "давайте", + }, + multi: []string{ + "хотел бы", "хотела бы", "хочу чтобы", "хотел чтобы", "хотела чтобы", + "могу ли", + }, +} + +// ── token helpers ───────────────────────────────────────────────────────── + +// tokens lowercases and splits on anything that is not a letter or digit, +// matching the router's planTokens discipline ("что-дальше" tokenises like +// "что дальше"). +func tokens(text string) []string { + return strings.FieldsFunc(strings.ToLower(text), func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsDigit(r) + }) +} + +func hasTok(toks []string, w string) bool { + for _, t := range toks { + if t == w { + return true + } + } + return false +} + +func hasAny(toks, ws []string) bool { + for _, w := range ws { + if hasTok(toks, w) { + return true + } + } + return false +} + +func indexTok(toks []string, w string) int { + for i, t := range toks { + if t == w { + return i + } + } + return -1 +} + +// isFiniteVerb reports a verb form that is not the dictionary (infinitive) +// form: "выключи" is finite, "выключить" is not. A finite verb at command +// position is positive request evidence; an infinitive is not. +func isFiniteVerb(tok string) bool { + if !morph.IsVerbForm(tok) { + return false + } + return morph.Lemma(tok) != tok +} + +// isInfinitive reports a token that morph resolves to its own dictionary form +// (the lemma ends in the infinitive ending by construction). +func isInfinitive(tok string) bool { + if !morph.IsVerbForm(tok) { + return false + } + return morph.Lemma(tok) == tok +} + +func anyInfinitive(toks []string) bool { + for _, t := range toks { + if isInfinitive(t) { + return true + } + } + return false +} + +func anyFiniteVerb(toks []string) bool { + for _, t := range toks { + if isFiniteVerb(t) { + return true + } + } + return false +} + +func hasAnyVerb(toks []string) bool { + for _, t := range toks { + if morph.IsVerbForm(t) { + return true + } + } + return false +} + +func blocked(rs ...Reason) Frame { return Frame{Eligibility: Blocked, Reasons: rs} } +func ambiguous(rs ...Reason) Frame { + return Frame{Eligibility: Ambiguous, Reasons: rs} +} +func permissive(rs ...Reason) Frame { + return Frame{Eligibility: Permissive, Reasons: rs} +} + +// ── quoted spans ───────────────────────────────────────────────────────── + +// quotedSpan is a maximal quoted interval in the normalized text. +type quotedSpan struct{ content string } + +// quotePairs covers the quoting styles the dev pool and brief fixtures use: +// Russian guillemets, curly double/single quotes, and straight quotes. +var quotePairs = []struct{ open, close string }{ + {"«", "»"}, {"„", "\""}, {"“", "”"}, {"‚", "‘"}, {"‘", "’"}, {"'", "'"}, {"\"", "\""}, +} + +// extractQuotedSpans returns the contents of quoted spans in order, in rune +// index space (the text is normalized, so glyphs are single runes). An +// unbalanced delimiter yields no span (best-effort; the conservative +// fallback then applies). +func extractQuotedSpans(t string) []quotedSpan { + runes := []rune(t) + var out []quotedSpan + i := 0 + for i < len(runes) { + matched := false + for _, p := range quotePairs { + po := []rune(p.open) + pc := []rune(p.close) + if i+len(po) > len(runes) || string(runes[i:i+len(po)]) != p.open { + continue + } + j := i + len(po) + for j+len(pc) <= len(runes) && string(runes[j:j+len(pc)]) != p.close { + j++ + } + out = append(out, quotedSpan{content: string(runes[i+len(po) : j])}) + i = j + len(pc) + matched = true + break + } + if !matched { + i++ + } + } + return out +} + +// ── rule stages (evaluated in this order; a decision is final) ──────────── + +// Evaluate derives the execution-frame verdict for one utterance. +func Evaluate(text string) Frame { + t := router.NormalizeMatchText(text) + if strings.TrimSpace(t) == "" { + return ambiguous(ReasonNoRequestEvidence) + } + toks := tokens(t) + joined := strings.Join(toks, " ") + + // 1. Quotation: a command inside a quoted span is not a command being + // issued now. With a reporting frame outside it is a quotation; a bare + // quote is at best ambiguous. + if f, ok := stageQuotation(t, toks, joined); ok { + return f + } + + // 2. Reported speech: a past/third-person report verb governing a command + // clause reports an order to someone else, it does not issue one. + if f, ok := stageReport(t, toks, joined); ok { + return f + } + + // 3. Hypothetical: a command scope opened by "если/if" that does not + // continue as a real condition→command is not an execution request. + if f, ok := stageHypothesis(toks, joined); ok { + return f + } + + // 4. Direct negative commands: the shipped prohibition parser. + if router.IsCommandProhibition(t) { + return blocked(ReasonCommandProhibition) + } + + // 5. Advisory negatives: "не надо/не стоит/не нужно …". + if f, ok := stageAdvisoryNegation(toks); ok { + return f + } + + // 6. Capability and permission modality (the measured core). + if f, ok := stageCapability(toks, joined); ok { + return f + } + + // 7. Trailing question mark with no modal at play: an uncertain posture, + // never a confirmed executable request. + if strings.HasSuffix(t, "?") { + return ambiguous(ReasonAmbiguousModal) + } + + // 8. Positive request evidence. + if hasRequestEvidence(toks, joined) { + return permissive(ReasonExplicitRequest) + } + + // 9. No execution pressure at all. + return ambiguous(ReasonNoRequestEvidence) +} + +// stageQuotation blocks a quoted command when a reporting frame surrounds it. +func stageQuotation(t string, toks []string, joined string) (Frame, bool) { + spans := extractQuotedSpans(t) + if len(spans) == 0 { + return Frame{}, false + } + commandSpan := false + for _, sp := range spans { + if isCommandishWithin(tokens(sp.content), strings.Join(tokens(sp.content), " ")) { + commandSpan = true + break + } + } + if !commandSpan { + return Frame{}, false + } + reasons := []Reason{ReasonQuotation} + if hasReportFrame(toks, joined) { + reasons = append(reasons, ReasonReportedSpeech) + return Frame{Eligibility: Blocked, Reasons: reasons}, true + } + // a bare quoted command has no reporting frame: refusable but not a + // definite prohibition either (it is at least ambiguous) + return Frame{Eligibility: Ambiguous, Reasons: reasons}, true +} + +// isCommandishWithin reports the span content carrying command or capability +// polarity itself — imperative, prohibition, or a can-form. +func isCommandishWithin(toks []string, joined string) bool { + if len(toks) == 0 { + return false + } + if router.IsCommandProhibition(strings.Join(toks, " ")) { + return true + } + if hasAny(toks, ruCanForms) || ruAbilityForms.in(toks, joined) || hasAny(toks, enCanForms) { + return true + } + return anyFiniteVerb(toks) +} + +func hasReportFrame(toks []string, joined string) bool { + if hasAny(toks, reportVerbs) { + return true + } + return hasAny(toks, reportNouns) +} + +// stageReport blocks when a report frame governs a command clause: an +// infinitive after the report verb, or a quoted imperative. Second-person +// imperatives like "скажи/расскажи" are not in reportVerbs, so a request to +// report ("расскажи мне, что сказал папа") passes through. +func stageReport(t string, toks []string, joined string) (Frame, bool) { + if !hasReportFrame(toks, joined) { + return Frame{}, false + } + last := -1 + for i, w := range toks { + if hasTok(reportVerbs, w) || hasTok(reportNouns, w) { + last = i + } + } + if last < 0 { + return Frame{}, false + } + after := toks[last+1:] + if len(after) == 0 { + return Frame{}, false + } + // a quoted command after the frame counts as the governed clause + for _, sp := range extractQuotedSpans(t) { + if isCommandishWithin(tokens(sp.content), strings.Join(tokens(sp.content), " ")) { + return blocked(ReasonReportedSpeech, ReasonQuotation), true + } + } + if anyInfinitive(after) || hasAny(after, []string{"что", "чтобы", "чтоб"}) { + return blocked(ReasonReportedSpeech), true + } + return Frame{}, false +} + +// stageHypothesis blocks a conditional scope whose clauses are hypothetical +// (infinitive or subjunctive "бы") rather than a real condition→command. +// "если будет дождь, выключи полив" keeps its imperative continuation and +// passes through; it is a real conditional request, not a hypothetical. +func stageHypothesis(toks []string, joined string) (Frame, bool) { + idx := -1 + for _, m := range hypothesisMarkers { + if i := indexTok(toks, m); i >= 0 && (idx < 0 || i < idx) { + idx = i + } + } + if idx < 0 { + return Frame{}, false + } + post := toks[idx+1:] + if len(post) == 0 || hasTok(post, "бы") || anyInfinitive(post) { + return blocked(ReasonHypothetical), true + } + // a real condition clause is not hypothetical: «если будет дождь, + // выключи полив» is a request. The dev dict does not cover «будет», so + // the imperative is looked for anywhere, not just after the marker + // («выключи свет, если будет дождь»). + if !anyFiniteVerb(toks) { + return blocked(ReasonHypothetical), true + } + return permissive(ReasonExplicitRequest), true +} + +// stageAdvisoryNegation blocks "не надо/не нужно/не стоит/не следует …". +// (absent from the dev pool; covered by brief fixtures) +func stageAdvisoryNegation(toks []string) (Frame, bool) { + if len(toks) < 3 || toks[0] != "не" { + return Frame{}, false + } + if !hasTok(toks[1:2], "надо") && !hasTok(toks[1:2], "нужно") && + !hasTok(toks[1:2], "стоит") && !hasTok(toks[1:2], "следует") && + !hasTok(toks[1:2], "требуется") { + return Frame{}, false + } + rest := toks[2:] + if anyInfinitive(rest) || anyFiniteVerb(rest) || hasAnyVerb(rest) { + return blocked(ReasonNegatedCommand), true + } + return Frame{}, false +} + +// stageCapability encodes the measured modal matrix. Returns a decision when +// modality alone settles the frame. +func stageCapability(toks []string, joined string) (Frame, bool) { + // "не мог бы ты …, пожалуйста" style conditional politeness is ordinary + // modal politeness (the prohibition parser exempts it as such): a request. + for _, pref := range politeNegativeModal { + if strings.HasPrefix(joined, pref) { + return permissive(ReasonExplicitRequest), true + } + } + + // "… ли" directly after a can-form is a polar capability question: + // "могу ли я …", "можешь ли ты …", "умеешь ли ты …", "можно ли …". + // Checked before the modality arms so the polar reading wins. + if hasTok(toks, "ли") { + for i := 1; i < len(toks); i++ { + if toks[i] != "ли" { + continue + } + prev := toks[i-1] + if hasTok(ruCanForms, prev) || prev == "можно" || hasTok(ruAbilityForms.single, prev) { + return blocked(ReasonCapabilityQuestion), true + } + } + } + + ruAddr := hasAny(toks, ruAddress) + ruCan := hasAny(toks, ruCanForms) + ruAbil := ruAbilityForms.in(toks, joined) + enCan := hasAny(toks, enCanForms) + polite := politeness.in(toks, joined) + + // addressed Russian can-form: capability question, always blocked. + // ("ты можешь выключить свет, пожалуйста" included — 42/42 non-action.) + if ruAddr && (ruCan || ruAbil) { + return blocked(ReasonCapabilityQuestion), true + } + + // ability forms (future/conditional/умеешь) are capability even bare and + // even polite: "сможешь открыть окно, пожалуйста" is 7/7 non-action. + if ruAbil && !ruCan { + return blocked(ReasonCapabilityQuestion), true + } + + // bare Russian present can-form: politeness is the request marker. + if ruCan && !ruAddr { + if polite { + return Frame{}, false // modal-request positive evidence is found later + } + return ambiguous(ReasonAmbiguousModal), true + } + + // English can/could: "can you …, please" is a request (96/96 action in the + // dev pool; the frame wraps a Russian imperative). Without politeness it + // reads as a capability question and stays ambiguous. + if enCan && !ruAddr { + if polite { + return Frame{}, false // positive modal-request evidence later + } + return ambiguous(ReasonAmbiguousModal), true + } + + // "можно" (permission): "можно ли …" is a permission question; a bare + // "можно …" is a politeness-implicature request. + if hasTok(toks, "можно") { + if hasTok(toks, "ли") { + return blocked(ReasonCapabilityQuestion), true + } + return Frame{}, false + } + + // bare "могу": a self-capability statement, not a request. + if hasTok(toks, "могу") && !hasTok(toks, "ли") { + return ambiguous(ReasonAmbiguousModal), true + } + + return Frame{}, false +} + +// hasRequestEvidence is the positive permissive trigger, reached only after +// every block/ambiguity stage above has declined. +func hasRequestEvidence(toks []string, joined string) bool { + polite := politeness.in(toks, joined) + enCan := hasAny(toks, enCanForms) + + // 1. politeness + a verb (or an English modal) is explicit request + // evidence: "можешь выключить свет, пожалуйста", "can you останови …, + // please", "выключи свет, пожалуйста". + if polite && (hasAnyVerb(toks) || enCan) { + return true + } + + // 2. first-person illocution frame: "я хочу …", "мне нужно …". + if hasAny(toks, lexicon.FirstPerson()) && illocutionVerbs.in(toks, joined) { + return true + } + + // 3. impersonal need: "надо …", "нужно …", "пора …". + if hasAny(toks, []string{"надо", "нужно", "следует", "пора", "требуется", "придётся", "придется"}) { + return true + } + + // 4. permission-implicature request: "можно выключить свет". + if hasTok(toks, "можно") && !hasTok(toks, "ли") { + return true + } + + // 5. reminder request in the parser's own exemption scope: the + // prohibition parser declines «не забудь напомнить про свет» as a + // reminder, not a prohibition — carry that into a request. + if strings.HasPrefix(joined, "не забудь") && hasAny(toks, lexicon.ReminderVerbs()) { + return true + } + + // 6. leading finite verb (imperative or otherwise tensed verb at command + // position): "выключи свет", "покажи что запущено". Address and filler + // particles are stripped first, so "мавен, выключи свет" rides the same + // frame. + lead := toks + for len(lead) > 0 { + first := lead[0] + if !lexicon.IsFillerParticle(first) && !hasTok(wakeAddr, first) && !hasTok(ruAddress, first) { + break + } + lead = lead[1:] + } + if len(lead) > 0 && isFiniteVerb(lead[0]) { + return true + } + + return false +} diff --git a/cmd/semantic-router-experiment/slice21/guard_test.go b/cmd/semantic-router-experiment/slice21/guard_test.go new file mode 100644 index 0000000..b7dfba7 --- /dev/null +++ b/cmd/semantic-router-experiment/slice21/guard_test.go @@ -0,0 +1,139 @@ +package main + +import ( + "encoding/json" + "os" + "strings" + "testing" + + "github.com/kami/maven/internal/morph" +) + +func TestEvaluateFixtures(t *testing.T) { + for _, fx := range Fixtures { + got := Evaluate(fx.Utterance) + if got.Eligibility != fx.Want { + t.Errorf("%s: want %s got %s (reasons %v)", fx.Utterance, fx.Want, got.Eligibility, got.Reasons) + } + } +} + +// TestMeasuredDiscriminators pins the corpus-verified numbers on the frozen +// pool. These are the exact measurements the rules were built on, so a change +// that moves them is a rule regression visible in the slice. +func loadPool(t *testing.T) []Row { + t.Helper() + b, err := os.ReadFile("/tmp/mvn-s21/pool.json") + if os.IsNotExist(err) { + t.Skip("pool.json missing; run slice21_emit.py first") + } + if err != nil { + t.Fatal(err) + } + var rows []Row + if err := json.Unmarshal(b, &rows); err != nil { + t.Fatal(err) + } + return rows +} + +func TestDevCapabilityProhibition(t *testing.T) { + if !morph.Available() { + t.Skip("morph dict unavailable") + } + rows := loadPool(t) + var n, pass, blockedN, ambig int + for _, r := range rows { + if !hasTok(r.Tags, "capability_question") { + continue + } + n++ + switch Evaluate(variants(r.NText)[vOrig]).Eligibility { + case Permissive: + pass++ + case Blocked: + blockedN++ + case Ambiguous: + ambig++ + } + } + if n != 126 { + t.Fatalf("capability-question rows = %d, want 126", n) + } + if pass != 0 { + t.Fatalf("capability-question dangerous pass = %d, want 0", pass) + } + if blockedN != 126 && ambig != 0 { + t.Errorf("blocked=%d ambig=%d, expect all 126 blocked", blockedN, ambig) + } +} + +func TestDevBareCanPoliteIsAction(t *testing.T) { + if !morph.Available() { + t.Skip("morph dict unavailable") + } + rows := loadPool(t) + total, action, perm := 0, 0, 0 + for _, r := range rows { + toks := tokens(r.NText) + if !hasTok(toks, "можешь") || hasAny(toks, ruAddress) || + hasAny(toks, append([]string{}, ruAbilityForms.single...)) { + continue + } + total++ + if r.Route == "action" { + action++ + } + if Evaluate(r.NText).Eligibility == Permissive { + perm++ + } + } + if total != 127 || action != 127 { + t.Fatalf("bare-можешь+polite: n=%d action=%d, want 127/127", total, action) + } + if perm != 127 { + t.Fatalf("bare-можешь+polite permissive=%d, want 127", perm) + } +} + +func TestDevEnglishCanPoliteIsAction(t *testing.T) { + if !morph.Available() { + t.Skip("morph dict unavailable") + } + rows := loadPool(t) + total, action, perm := 0, 0, 0 + for _, r := range rows { + toks := tokens(r.NText) + if !hasAny(toks, enCanForms) { + continue + } + total++ + if r.Route == "action" { + action++ + } + if Evaluate(r.NText).Eligibility == Permissive { + perm++ + } + } + if total != 96 || action != 96 { + t.Fatalf("can-rows: n=%d action=%d, want 96/96", total, action) + } + if perm != 96 { + t.Fatalf("can-rows permissive=%d, want 96", perm) + } +} + +func TestNoPermissiveRowEndsInQuestion(t *testing.T) { + // asymmetric posture: the guard must never approve a trailing-? frame -- + // the corpus has 0 action rows ending in "?", and approving any would + // bet on punctuation the slice has ruled uncertain. + rows := loadPool(t) + for _, r := range rows { + if !strings.HasSuffix(r.NText, "?") { + continue + } + if got := Evaluate(r.NText).Eligibility; got == Permissive { + t.Errorf("canonical '?': %q approved (%s)", r.Text, got) + } + } +} diff --git a/cmd/semantic-router-experiment/slice21/main.go b/cmd/semantic-router-experiment/slice21/main.go new file mode 100644 index 0000000..9c9e657 --- /dev/null +++ b/cmd/semantic-router-experiment/slice21/main.go @@ -0,0 +1,566 @@ +// Slice 21 runner: report the deterministic execution-frame guard against the +// frozen slice-20 dev pool. +// +// Reads the emit step's compact files (pool.json, pairs.json, sparse_oof.json +// in /tmp/mvn-s21) and prints the report tables plus a machine-readable +// guard_results.json. Reuses router/morph/lexicon parsers live inside this +// module — the pool texts are the only data, no embedding is recomputed. +// +// Usage: go run ./cmd/semantic-router-experiment/slice21 +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "regexp" + "sort" + "strings" + "unicode" +) + +const sparseThreshold = 0.715 // slice-18 §4 strict operating point (P>=0.95 best recall) + +var familyPriority = []string{ + "capability_question", "question", "first_person_request", + "modal_request", "polite_request", "reordered_target", "direct_imperative", +} + +func familyOf(tags []string) string { + for _, f := range familyPriority { + if hasTok(tags, f) { + return f + } + } + return "other" +} + +// ── pool row ────────────────────────────────────────────────────────────── + +type Row struct { + Idx int `json:"idx"` + Text string `json:"text"` + NText string `json:"n_text"` + Route string `json:"route"` + Y int `json:"y"` + Tags []string `json:"tags"` + CVFold int `json:"cv_fold"` + SplitGp string `json:"split_group"` + SourceID string `json:"source_id"` + Family string + VariantOf int +} + +type Pair struct{ Cap, Act int } + +// ── stress variants ─────────────────────────────────────────────────────── + +var nofinalRe = regexp.MustCompile(`[?.!,;:]+$`) + +// strip_punct mirrors the slice-18/19 python strip_punct: trailing sentence +// punctuation, then every non-word/non-space rune. +func stripPunct(t string) string { + t = nofinalRe.ReplaceAllString(strings.TrimSpace(t), "") + out := make([]rune, 0, len(t)) + var prevSpace bool + for _, r := range t { + if unicode.IsLetter(r) || unicode.IsNumber(r) { + out = append(out, r) + prevSpace = false + } else if !prevSpace { + out = append(out, ' ') + prevSpace = true + } + } + return strings.TrimSpace(string(out)) +} + +func variants(nText string) [3]string { + return [3]string{ + nText, + nofinalRe.ReplaceAllString(strings.TrimSpace(nText), ""), + stripPunct(nText), + } +} + +const ( + vOrig = iota + vNofinal + vStrip +) + +var variantName = [3]string{"orig", "nofinal", "strip"} + +type result struct { + Frame Frame `json:"frame"` +} + +// ── metrics ────────────────────────────────────────────────────────────── + +type triTab struct { + Permissive, Blocked, Ambiguous int + PermNonact, BlockedAction, AmbAction int +} + +type runAgg struct { + n, action, tp, fp, fn int + approved int + capPermissive int +} + +func (a *runAgg) addApproved(approved bool, route string) { + a.n++ + if route == "action" { + a.action++ + } + if approved { + a.approved++ + if route == "action" { + a.tp++ + } else { + a.fp++ + } + } else if route == "action" { + a.fn++ + } +} + +func (a *runAgg) P() string { return fmtPct(frac(a.tp, a.tp+a.fp)) } +func (a *runAgg) R() string { return fmtPct(frac(a.tp, a.action)) } +func (a *runAgg) FA() int { return a.fp } +func (a *runAgg) FArate() string { + return fmtPct(frac(a.fp, a.n)) +} + +func maxi(a, b int) int { + if a > b { + return a + } + return b +} + +// frac is the guarded ratio the tables print (0/0 is 0). +func frac(num, den int) float64 { return float64(num) / float64(maxi(den, 1)) } + +func fmtPct(v float64) string { return fmt.Sprintf("%.1f%%", 100*v) } + +// ── main ───────────────────────────────────────────────────────────────── + +func main() { + poolPath := flag.String("pool", "/tmp/mvn-s21/pool.json", "dev pool rows") + pairsPath := flag.String("pairs", "/tmp/mvn-s21/pairs.json", "cap-vs-action pairs") + sparsePath := flag.String("sparse", "/tmp/mvn-s21/sparse_oof.json", "slice-18 both OOF proba") + outPath := flag.String("out", "/tmp/mvn-s21/guard_results.json", "machine-readable results") + flag.Parse() + + rows := mustLoad[[]Row](*poolPath) + // pairs.json is bare [cap, act] index pairs; adapt into typed pairs. + rawPairs := mustLoad[[][2]int](*pairsPath) + pairs := make([]Pair, 0, len(rawPairs)) + for _, rp := range rawPairs { + pairs = append(pairs, Pair{Cap: rp[0], Act: rp[1]}) + } + sparseOOF := mustLoad[[]struct { + Idx int `json:"idx"` + Proba float64 `json:"proba"` + }](*sparsePath) + proba := make([]float64, len(rows)) + for _, s := range sparseOOF { + proba[s.Idx] = s.Proba + } + + famPrio := 0 + for i := range rows { + rows[i].Family = familyOf(rows[i].Tags) + if rows[i].Family != "other" { + famPrio++ + } + } + _ = famPrio + + // verdicts per variant + type rowRes struct { + Idx int `json:"idx"` + Text string `json:"text"` + Route string `json:"route"` + Family string `json:"family"` + Tags []string `json:"tags"` + Frames map[string]string `json:"frames"` // variant -> eligibility + } + + perVariant := make([][3]Frame, len(rows)) + fmt.Println("slice 21 — deterministic execution-frame guard on slice-20 dev pool") + fmt.Println("==================================================================") + + for i, r := range rows { + vs := variants(r.NText) + var fr [3]Frame + for vi := 0; vi < 3; vi++ { + fr[vi] = Evaluate(vs[vi]) + } + perVariant[i] = fr + } + + // ── §3 three-way cross-tab (orig) ───────────────────────────────────── + fmt.Println("\n## 1. Three-way eligibility × route (orig)") + tab := triTab{} + for i, r := range rows { + switch perVariant[i][vOrig].Eligibility { + case Permissive: + tab.Permissive++ + if r.Route != "action" { + tab.PermNonact++ + } + case Blocked: + tab.Blocked++ + if r.Route == "action" { + tab.BlockedAction++ + } + case Ambiguous: + tab.Ambiguous++ + if r.Route == "action" { + tab.AmbAction++ + } + } + } + fmt.Printf("permissive: %d blocked: %d ambiguous: %d\n", tab.Permissive, tab.Blocked, tab.Ambiguous) + fmt.Printf(" permissive non-action: %d blocked action: %d ambiguous action: %d\n", + tab.PermNonact, tab.BlockedAction, tab.AmbAction) + + // ── §4 binary executable-gate metrics on orig ───────────────────────── + fmt.Println("\n## 2. Binary executable gate (approve = permissive; deny = blocked|ambiguous)") + a := runAgg{} + capCov, capPerm, capAmb := 0, 0, 0 + for i, r := range rows { + el := perVariant[i][vOrig].Eligibility + a.addApproved(el == Permissive, r.Route) + if hasTok(r.Tags, "capability_question") { + capCov++ + switch el { + case Permissive: + capPerm++ + case Ambiguous: + capAmb++ + } + } + } + fmt.Printf("approved: %d denied: %d (n=%d, action=%d)\n", a.approved, a.n-a.approved, a.n, a.action) + fmt.Printf("action precision %s recall %s FA %d (%s)\n", a.P(), a.R(), a.FA(), a.FArate()) + fmt.Printf("capability-question dangerous pass: %d / %d (rate %s)\n", + capPerm, capCov, fmtPct(frac(capPerm, capCov))) + fmt.Printf("capability-question blocked %d, ambiguous %d\n", capCov-capPerm-capAmb, capAmb) + + // ── §15 family stress (orig) ────────────────────────────────────────── + fmt.Println("\n## 3. Family stress (orig; counts per eligibility)") + fmt.Printf("%-24s %8s %8s %8s %8s\n", "family", "n", "perm", "block", "ambig") + famOrder := []string{"direct_imperative", "polite_request", "modal_request", "first_person_request", + "reordered_target", "capability_question", "question", "other"} + famAgg := map[string]*triTab{} + for _, f := range famOrder { + famAgg[f] = &triTab{} + } + for i, r := range rows { + t := famAgg[r.Family] + if t == nil { + continue + } + switch perVariant[i][vOrig].Eligibility { + case Permissive: + t.Permissive++ + if r.Route != "action" { + t.PermNonact++ + } + case Blocked: + t.Blocked++ + case Ambiguous: + t.Ambiguous++ + } + } + for _, f := range famOrder { + t := famAgg[f] + if t == nil { + continue + } + n := t.Permissive + t.Blocked + t.Ambiguous + if n == 0 { + continue + } + fmt.Printf("%-24s %8d %8d %8d %8d\n", f, n, t.Permissive, t.Blocked, t.Ambiguous) + } + + // ── §cap-Q LOFO across stress variants ──────────────────────────────── + fmt.Println("\n## 4. Capability-question dangerous pass by stress variant") + for vi := 0; vi < 3; vi++ { + cp, cb, ca := 0, 0, 0 + for i, r := range rows { + if !hasTok(r.Tags, "capability_question") { + continue + } + switch perVariant[i][vi].Eligibility { + case Permissive: + cp++ + case Blocked: + cb++ + case Ambiguous: + ca++ + } + } + fmt.Printf(" %-8s dangerous-pass %d blocked %d ambiguous %d\n", + variantName[vi], cp, cb, ca) + } + + // ── §pair test ──────────────────────────────────────────────────────── + fmt.Println("\n## 5. Paired action/capability (cap row must never clear)") + capClear, actPerm, actAmbig, actBlock := 0, 0, 0, 0 + for _, p := range pairs { + cel := perVariant[p.Cap][vOrig].Eligibility + ael := perVariant[p.Act][vOrig].Eligibility + if cel == Permissive { + capClear++ + } + switch ael { + case Permissive: + actPerm++ + case Ambiguous: + actAmbig++ + case Blocked: + actBlock++ + } + } + fmt.Printf("pairs %d: cap cleared %d (rate %s), action permissive %d, action ambiguous %d, action blocked %d\n", + len(pairs), capClear, fmtPct(frac(capClear, len(pairs))), + actPerm, actAmbig, actBlock) + + // ── �safe composition §17 ───────────────────────────────────────────── + fmt.Println("\n## 6. Composition: guard-alone / sparse-alone / guard→sparse (orig)") + compose := map[string]*runAgg{ + "guard_alone": {}, + "sparse_alone": {}, + "guard_sparse": {}, + } + for i, r := range rows { + gPerm := perVariant[i][vOrig].Eligibility == Permissive + sPerm := proba[i] >= sparseThreshold + compose["guard_alone"].addApproved(gPerm, r.Route) + compose["sparse_alone"].addApproved(sPerm, r.Route) + compose["guard_sparse"].addApproved(gPerm && sPerm, r.Route) + } + fmt.Printf("%-14s %8s %8s %6s %10s %6s %10s\n", "policy", "P", "R", "FA", "FA rate", "capQ", "capQ rate") + for _, name := range []string{"guard_alone", "sparse_alone", "guard_sparse"} { + agg := compose[name] + capQ := 0 + for i, r := range rows { + if !hasTok(r.Tags, "capability_question") { + continue + } + ok := false + switch name { + case "guard_alone": + ok = perVariant[i][vOrig].Eligibility == Permissive + case "sparse_alone": + ok = proba[i] >= sparseThreshold + case "guard_sparse": + ok = perVariant[i][vOrig].Eligibility == Permissive && proba[i] >= sparseThreshold + } + if ok { + capQ++ + } + } + fmt.Printf("%-14s %8s %8s %6d %10s %6d %10s\n", name, agg.P(), agg.R(), agg.FA(), + agg.FArate(), capQ, fmtPct(float64(capQ)/126)) + } + + // composition on strip too (brief §16 voice stress) + fmt.Println("\n## 7. Composition on punctuation-stripped text (strip)") + c2 := runAgg{} + capQ2 := 0 + for i, r := range rows { + gPerm := perVariant[i][vStrip].Eligibility == Permissive + ok := gPerm && proba[i] >= sparseThreshold + c2.addApproved(ok, r.Route) + if hasTok(r.Tags, "capability_question") && ok { + capQ2++ + } + } + fmt.Printf("guard→sparse strip: P %s R %s FA %d (%s) capQ pass %d\n", + c2.P(), c2.R(), c2.FA(), c2.FArate(), capQ2) + + // ── §19 manual classification scratch ───────────────────────────────── + fmt.Println("\n## 8. Manual classification (scan material written to manual_class.json)") + var dangerous []map[string]any + var permNonact []map[string]any + var deniedAction []map[string]any + for i, r := range rows { + fr := perVariant[i][vOrig] + if hasTok(r.Tags, "capability_question") && fr.Eligibility == Permissive { + dangerous = append(dangerous, map[string]any{ + "idx": r.Idx, "text": r.Text, "route": r.Route, + "reasons": fr.Reasons, + }) + } + if fr.Eligibility == Permissive && r.Route != "action" { + permNonact = append(permNonact, map[string]any{ + "idx": r.Idx, "text": r.Text, "route": r.Route, + "family": r.Family, "reasons": fr.Reasons, + }) + } + if fr.Eligibility != Permissive && r.Route == "action" { + deniedAction = append(deniedAction, map[string]any{ + "idx": r.Idx, "text": r.Text, "family": r.Family, + "eligibility": fr.Eligibility.String(), "reasons": fr.Reasons, + }) + } + } + writeManual(permNonact, deniedAction, dangerous) + fmt.Printf("dangerous passes: %d permissive non-action: %d denied action: %d\n", + len(dangerous), len(permNonact), len(deniedAction)) + groupAndSample("permissive non-action by reason+family", permNonact, 4) + groupAndSample("denied action by reason+family", deniedAction, 4) + + // ── fixtures ────────────────────────────────────────────────────────── + fmt.Println("\n## 9. Brief fixtures") + pass := 0 + for _, fx := range Fixtures { + got := Evaluate(fx.Utterance) + mark := "ok " + if got.Eligibility != fx.Want { + mark = "FAIL" + } else { + pass++ + } + if got.Eligibility != fx.Want { + fmt.Printf(" %s %-14s want %-10s got %-10s %s\n", mark, fx.Family, + fx.Want, got.Eligibility.String(), fx.Utterance) + } + } + fmt.Printf("fixtures: %d/%d passed\n", pass, len(Fixtures)) + + // write result file + rr := make([]rowRes, 0, len(rows)) + for i, r := range rows { + fr := [3]string{"", "", ""} + for vi := 0; vi < 3; vi++ { + fr[vi] = perVariant[i][vi].Eligibility.String() + } + rr = append(rr, rowRes{ + Idx: r.Idx, Text: r.Text, Route: r.Route, Family: r.Family, Tags: r.Tags, + Frames: map[string]string{ + "orig": fr[vOrig], "nofinal": fr[vNofinal], "strip": fr[vStrip], + }, + }) + } + if *outPath != "" { + mustSave(*outPath, map[string]any{ + "pool": "/tmp/mvn-s21/pool.json", + "rows": rr, + "aggregates": map[string]any{ + "tab": tab, + "capq_pass": capPerm, + "capq_blocked": capCov - capPerm - capAmb, + "capq_ambiguous": capAmb, + "binary": map[string]any{"tp": a.tp, "fp": a.fp, "fn": a.fn, "approved": a.approved, "n": a.n}, + "pairs": map[string]any{"n": len(pairs), "cap_cleared": capClear, "act_permissive": actPerm}, + "guard_sparse": map[string]any{"tp": compose["guard_sparse"].tp, "fp": compose["guard_sparse"].fp, "fn": compose["guard_sparse"].fn}, + "dangerous_passes": len(dangerous), + "perm_nonact_count": len(permNonact), + "denied_action": len(deniedAction), + }, + }) + fmt.Println("wrote", *outPath) + } +} + +// ── manual classification helpers ───────────────────────────────────────── + +func writeManual(permNonact, deniedAction, dangerous []map[string]any) { + writeJSON("/tmp/mvn-s21/manual_class.json", map[string]any{ + "dangerous_passes": dangerous, + "permissive_non_action": permNonact, + "denied_action": deniedAction, + }) +} + +func groupAndSample(title string, rows []map[string]any, sample int) { + type g struct { + key string + n int + texts []string + } + groups := map[string]*g{} + var order []string + for _, r := range rows { + var family, reason, el string + if v, ok := r["family"].(string); ok { + family = v + } + if v, ok := r["eligibility"].(string); ok { + el = v + } + if rs, ok := r["reasons"].([]Reason); ok { + rs2 := make([]string, len(rs)) + for k, rr := range rs { + rs2[k] = rr.String() + } + reason = strings.Join(rs2, ",") + } else if rs, ok := r["reasons"].([]string); ok { + reason = strings.Join(rs, ",") + } + key := fmt.Sprintf("family=%s elig=%s reason=%s", family, el, reason) + if _, ok := groups[key]; !ok { + groups[key] = &g{key: key} + order = append(order, key) + } + groups[key].n++ + if len(groups[key].texts) < sample { + groups[key].texts = append(groups[key].texts, firstN(fmt.Sprint(r["text"]), 60)) + } + } + fmt.Printf("%s (%d rows):\n", title, len(rows)) + for _, key := range order { + gr := groups[key] + fmt.Printf(" %-58s n=%d %s\n", gr.key, gr.n, strings.Join(gr.texts, " | ")) + } +} + +func firstN(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "…" +} + +// ── io helpers ──────────────────────────────────────────────────────────── + +func mustLoad[T any](path string) T { + b, err := os.ReadFile(path) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + var v T + if err := json.Unmarshal(b, &v); err != nil { + fmt.Fprintln(os.Stderr, "json:", err) + os.Exit(1) + } + return v +} + +func mustSave(path string, v any) { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err := os.WriteFile(path, b, 0o644); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func writeJSON(path string, v any) { + b, _ := json.MarshalIndent(v, "", " ") + _ = os.WriteFile(path, b, 0o644) +} + +var _ = sort.Strings diff --git a/cmd/semantic-router-experiment/slice21_emit.py b/cmd/semantic-router-experiment/slice21_emit.py new file mode 100644 index 0000000..4a1e257 --- /dev/null +++ b/cmd/semantic-router-experiment/slice21_emit.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +""" +Slice 21 emit: deterministic execution-frame guard — data files for the Go harness +================================================================================== + +Slice 18 showed the sparse lexical gate owns the aggregate boundary (PR-AUC +0.838, strict operating point at threshold 0.715 with P>=0.95 | R=0.264) and +slice 19/20 showed learning heads collapse on capability-question LOFO. Slice 21 +tests the deterministic alternative: a rule engine over existing parsers that +decides execution eligibility as a three-way gate (permissive / blocked / +ambiguous), never itself routing. + +This script only repackages the frozen dev pool for the Go harness. It reuses +slice 18's feature builders and grouped-CV and slice 19's pair builder verbatim, +so the numbers the Go side reports are the same populations the accepts +measured. It writes: + + /tmp/mvn-s21/pool.json dev rows: idx, text, n_text, route, y, tags, + cv_fold, split_group, source_id, family + /tmp/mvn-s21/pairs.json capability-vs-action pairs (slice-19 builder) + /tmp/mvn-s21/sparse_oof.json slice-18 "both" grouped-CV OOF proba per row + (for the §17 guard+sparse composition) + +No training happens here and no label is changed. The guard itself is Go. +""" + +import json +import os +import re +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) + +import slice18_sparse # noqa: E402 +import slice19_main # noqa: E402 + +OUT_DIR = "/tmp/mvn-s21" +SPARSE_THRESHOLD = 0.715 # slice-18 §4 strict-operating-point (P>=0.95 best recall) + + +def main(): + # Population = the exact slice-20 dev pool (s19.load_dev): every dev_pool + # row, fast-path included. The guard is evaluated on what slice 20 measured. + meta, examples = slice18_sparse.load_data() + dev = slice18_sparse.filter_dev_pool(examples) + print(f"dev pool (all dev_pool rows): {len(dev)} rows") + print(f"corpus meta: {meta.get('dev_count', '?')} dev rows declared, " + f"{meta.get('route_counts', {}).get('action', '?')} action declared") + + n_texts = [slice18_sparse.normalize_match_text(e["text"]) for e in dev] + + rows = [] + by_route = {} + by_family = {} + for i, (e, nt) in enumerate(zip(dev, n_texts)): + tags = sorted(set(e.get("tags", []))) + route = e["route"] + fam = slice19_main.family_of(set(tags)) + by_route[route] = by_route.get(route, 0) + 1 + by_family[fam] = by_family.get(fam, 0) + 1 + rows.append({ + "idx": i, + "text": e["text"], + "n_text": nt, + "route": route, + "y": 1 if route == "action" else 0, + "tags": tags, + "cv_fold": e["cv_fold"], + "split_group": e["split_group"], + "source_id": e["source_id"], + }) + + print("routes:", by_route) + print("families:", by_family) + + # ── pairs (slice-19 builder, exact population) ───────────────────────── + ldev = [{ + "text_orig": nt, + "route": r["route"], + "y": r["y"], + "cv_fold": r["cv_fold"], + "tags": set(r["tags"]), + "source_id": r["source_id"], + } for r, nt in zip(rows, n_texts)] + pairs = slice19_main.build_pairs(ldev, n_texts) + print(f"pairs: {len(pairs)}") + + # ── slice-18 "both" grouped-CV OOF proba, aligned to row index ──────── + y = [1 if r["route"] == "action" else 0 for r in rows] + folds = [r["cv_fold"] for r in rows] + X, _vec = slice18_sparse.build_features(n_texts, "both") + print(f"sparse 'both' X: {X.shape}") + yb = np.array(y) + folds_arr = np.array(folds) + idx_proba = {} + for te_fold in sorted(set(folds)): + tr = folds_arr != te_fold + te = folds_arr == te_fold + clf = slice18_sparse.LogisticRegression( + C=1.0, max_iter=2000, solver="lbfgs", random_state=42) + clf.fit(X[tr], yb[tr]) + p = clf.predict_proba(X[te])[:, 1] + te_idx = np.where(te)[0] + for k, i in enumerate(te_idx): + idx_proba[int(i)] = float(p[k]) + assert len(idx_proba) == len(rows) + sparse_oof = [{"idx": i, "proba": idx_proba[i]} for i in range(len(rows))] + pred = [1 if idx_proba[i] >= 0.5 else 0 for i in range(len(rows))] + tp = sum(1 for i in range(len(rows)) if y[i] == 1 and pred[i] == 1) + fp = sum(1 for i in range(len(rows)) if y[i] == 0 and pred[i] == 1) + fn = sum(1 for i in range(len(rows)) if y[i] == 1 and pred[i] == 0) + print(f"sparse both OOF @0.5: P={tp/max(tp+fp,1):.3f} R={tp/max(tp+fn,1):.3f} " + f"FA={fp} ({fp/len(rows):.4f})") + pred21 = [1 if idx_proba[i] >= SPARSE_THRESHOLD else 0 for i in range(len(rows))] + tp = sum(1 for i in range(len(rows)) if y[i] == 1 and pred21[i] == 1) + fp = sum(1 for i in range(len(rows)) if y[i] == 0 and pred21[i] == 1) + fn = sum(1 for i in range(len(rows)) if y[i] == 1 and pred21[i] == 0) + print(f"sparse both OOF @{SPARSE_THRESHOLD}: P={tp/max(tp+fp,1):.3f} " + f"R={tp/max(tp+fn,1):.3f} FA={fp} ({fp/len(rows):.4f})") + + os.makedirs(OUT_DIR, exist_ok=True) + with open(os.path.join(OUT_DIR, "pool.json"), "w") as f: + json.dump(rows, f, ensure_ascii=False, indent=1) + with open(os.path.join(OUT_DIR, "pairs.json"), "w") as f: + json.dump([[c, a] for c, a in pairs], f) + with open(os.path.join(OUT_DIR, "sparse_oof.json"), "w") as f: + json.dump(sparse_oof, f) + print(f"wrote {OUT_DIR}/{{pool,pairs,sparse_oof}}.json") + + +if __name__ == "__main__": + main() \ No newline at end of file