Compare commits

...

1 Commits

Author SHA1 Message Date
claude afac8fb670 mavend: run the persona checks before she speaks (V-399)
The checks stay in the eval package and the daemon calls three of them:
feminine, address, and a new leaked-reasoning test. No retry — it doubles
the latency on the turn that is already going badly, and on the nudge path
the moment has passed. A failure falls back to the deterministic floor and
is logged with the whole rejected text and counted by check name.

hisgender is deliberately not run: the simulator showed it rejecting
"записала, что ты выпил воды", which is her own correct self-reference.
2026-08-04 04:35:42 +04:00
5 changed files with 230 additions and 0 deletions
+134
View File
@@ -0,0 +1,134 @@
package main
import (
"context"
"log"
"regexp"
"strings"
"sync"
"github.com/kami/maven/internal/delivery"
"github.com/kami/maven/internal/loop"
"github.com/kami/maven/internal/phraser"
"github.com/kami/maven/internal/phraser/eval"
)
// The persona checks, run before she speaks (Vikunja #399).
//
// RunChecks and RunTalkChecks only ever ran from the eval package, so
// everything the fixtures measured was offline knowledge: we could say "about
// one reply in three is broken" and still ship every one of them. This runs the
// cheap half of that on the live path, and replaces a failing message with the
// deterministic floor.
//
// Which checks: the unambiguous string tests only — feminine self-reference,
// how she addresses him, and a leaked-reasoning test. Not length, which is
// path-specific, and not ontopic, which compares against fragments the fixture
// supplies and runtime does not have. Not hisgender either — see guardSpoken.
//
// No retry. A retry doubles the latency on the exact turn that is already going
// badly, and on the nudge path the moment has passed.
//
// The known cost, written down because it is real: a wrongly flagged good reply
// is replaced by a flatter stub one. That is the right trade — a stub sentence
// is dull, a leaked reasoning trace is broken — but it means these checks can
// no longer be tuned for sensitivity alone.
// checkLeak — the name reported when the model's scaffolding reaches the text.
const checkLeak = "leak"
// leakPatterns — reasoning and protocol that belongs to the model, not to him.
// The resident model is a Thinking variant, so an unclosed reasoning block is
// the failure mode, not a hypothetical (Vikunja #398).
var leakPatterns = []*regexp.Regexp{
regexp.MustCompile(`(?i)<\s*/?\s*think`),
regexp.MustCompile(`(?i)thinking\s*(process|:)`),
regexp.MustCompile(`(?i)^\s*(assistant|user|system)\s*:`),
// Raw contract JSON: the parser already unwraps a good one, so a body that
// still carries the keys is one it could not read.
regexp.MustCompile(`"(response|mood|body|summary)"\s*:`),
// The persona block quoted back at him.
regexp.MustCompile(`(?i)(ты\s+—?\s*мэйвен|системный промпт|system prompt)`),
}
// checkPersonaLeak reports whether the model's own scaffolding is in the text.
func checkPersonaLeak(body string) (string, bool) {
for _, re := range leakPatterns {
if m := re.FindString(body); m != "" {
return "leaked " + strings.TrimSpace(m), false
}
}
return "", true
}
// personaRejects counts what the guard caught, by check name, so the real
// production rate is knowable rather than inferred from the fixture.
var personaRejects = struct {
mu sync.Mutex
by map[string]int
}{by: map[string]int{}}
func personaRejectCounts() map[string]int {
personaRejects.mu.Lock()
defer personaRejects.mu.Unlock()
out := make(map[string]int, len(personaRejects.by))
for k, v := range personaRejects.by {
out[k] = v
}
return out
}
// guardSpoken checks a phrased message. It returns the failed check and false
// when the message must not be said; path names the caller, for the log.
//
// An empty message passes: the caller already treats that as a failure and
// falls back on its own, and reporting it as a persona breach would put a
// misleading line in the count.
func guardSpoken(path, body string) (string, bool) {
if strings.TrimSpace(body) == "" {
return "", true
}
if detail, ok := checkPersonaLeak(body); !ok {
return rejectSpoken(path, checkLeak, detail, body), false
}
// Feminine and address only. HisGender is not run here: it reads a
// sentence-initial feminine verb with no pronoun — "записала, что ты выпил
// воды" — as a woman being addressed, when it is her own correct
// self-reference. Offline that is a point of score; on this path it would
// replace a good reply with a stub one on every fact she confirms.
for _, r := range []eval.Result{eval.Feminine(body), eval.Address(body)} {
if !r.Pass {
return rejectSpoken(path, r.Name, r.Detail, body), false
}
}
return "", true
}
// rejectSpoken logs what she nearly said and counts it. The whole text, not a
// prefix: the point of the log line is that the failure can be read back later
// and argued with.
func rejectSpoken(path, check, detail, body string) string {
personaRejects.mu.Lock()
personaRejects.by[check]++
personaRejects.mu.Unlock()
log.Printf("persona: %s rejected on %s (%s): %q", path, check, detail, body)
return check
}
// guardNudge checks a phrased nudge and falls back to the deterministic floor
// when it fails. The nudge path, unlike the reply path, cannot ask again: the
// tick has already decided she speaks, so the choice is the floor's wording or
// a broken sentence.
func guardNudge(pn delivery.PhrasedNudge, cand loop.Candidate) delivery.PhrasedNudge {
if _, ok := guardSpoken("nudge", pn.Body); ok {
return pn
}
stub, err := phraser.NewStub().PhraseNudge(context.Background(), cand)
if err != nil {
// The Stub is templates over the candidate and does not fail. If it
// somehow does, the model's text is still what the rule decided to
// say, and saying nothing is the worse outcome.
return pn
}
return stub
}
+75
View File
@@ -0,0 +1,75 @@
package main
import (
"strings"
"testing"
"github.com/kami/maven/internal/delivery"
"github.com/kami/maven/internal/loop"
)
func TestGuardPassesWhatSheShouldSay(t *testing.T) {
good := []string{
"записала: купить хлеб.",
"поняла, напомню в 11:00.",
"ты не пил воду с утра.",
"я рада, что получилось.",
"",
}
for _, body := range good {
if check, ok := guardSpoken("test", body); !ok {
t.Errorf("guardSpoken(%q) rejected on %s", body, check)
}
}
}
func TestGuardStopsWhatSheShouldNot(t *testing.T) {
bad := []struct {
body string
want string
}{
{"<think>он просил воду</think> попей воды.", checkLeak},
{"Thinking Process: он давно не пил.", checkLeak},
{`{"response": "попей воды", "mood": "neutral"}`, checkLeak},
{"я напомнил тебе про воду.", "feminine"},
{"вы давно не пили воду.", "address"},
}
for _, c := range bad {
check, ok := guardSpoken("test", c.body)
if ok {
t.Errorf("guardSpoken(%q) let it through", c.body)
continue
}
if check != c.want {
t.Errorf("guardSpoken(%q) failed on %s; want %s", c.body, check, c.want)
}
}
}
func TestGuardCountsWhatItCaught(t *testing.T) {
before := personaRejectCounts()[checkLeak]
if _, ok := guardSpoken("test", "<think>…"); ok {
t.Fatal("a leaked reasoning block was let through")
}
if after := personaRejectCounts()[checkLeak]; after != before+1 {
t.Errorf("leak count %d; want %d", after, before+1)
}
}
// TestGuardNudgeFallsBackToTheFloor — a broken nudge is replaced by the
// deterministic wording, not dropped and not retried.
func TestGuardNudgeFallsBackToTheFloor(t *testing.T) {
cand := loop.Candidate{Rule: loop.Rule{Name: "water"}}
bad := delivery.PhrasedNudge{Candidate: cand, Body: "Thinking Process: он не пил.", Mood: "neutral"}
got := guardNudge(bad, cand)
if got.Body == bad.Body {
t.Fatal("the broken nudge was delivered unchanged")
}
if strings.TrimSpace(got.Body) == "" {
t.Fatal("the nudge was dropped rather than re-worded")
}
good := delivery.PhrasedNudge{Candidate: cand, Body: "попей воды.", Mood: "neutral"}
if guardNudge(good, cand).Body != good.Body {
t.Error("a good nudge was replaced")
}
}
+5
View File
@@ -30,5 +30,10 @@ func (r *llmReplier) Reply(d router.Decision) string {
if err != nil || out == "" {
return r.stub.Reply(d)
}
// The persona checks, on the live path (personaguard.go). A reply that
// leaks reasoning or calls him "вы" is worse than a flat one.
if _, ok := guardSpoken("reply", out); !ok {
return r.stub.Reply(d)
}
return out
}
+3
View File
@@ -176,6 +176,9 @@ func (t *tickLoop) tick(ctx context.Context, now time.Time) {
t.queueNudge(ctx, cand, state, now)
} else {
pn, err := t.phraser.PhraseNudge(ctx, *cand)
if err == nil {
pn = guardNudge(pn, *cand)
}
if err != nil {
log.Printf("tick: phrase nudge %s: %v", cand.Rule.Name, err)
} else {
+13
View File
@@ -67,6 +67,19 @@ func RunChecks(c Case, body, mood string) []Result {
}
}
// Feminine, HisGender and Address expose three checks one at a time, so the
// daemon can run them on a phrased message before he hears it (Vikunja #399).
// Only these three: they are unambiguous string tests with nothing to compare
// against, while length is path-specific and ontopic needs the fixture's
// expected fragments, which do not exist at runtime.
func Feminine(body string) Result { return checkFeminine(body) }
// HisGender — see checkHisGender.
func HisGender(body string) Result { return checkHisGender(body) }
// Address — see checkAddress.
func Address(body string) Result { return checkAddress(body) }
func checkMood(mood string) Result {
if Moods[mood] {
return Result{CheckMood, true, ""}