afac8fb670
The checks stay in the eval package and the daemon calls three of them: feminine, address, and a new leaked-reasoning test. No retry — it doubles the latency on the turn that is already going badly, and on the nudge path the moment has passed. A failure falls back to the deterministic floor and is logged with the whole rejected text and counted by check name. hisgender is deliberately not run: the simulator showed it rejecting "записала, что ты выпил воды", which is her own correct self-reference.
135 lines
5.1 KiB
Go
135 lines
5.1 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
"regexp"
|
|
"strings"
|
|
"sync"
|
|
|
|
"github.com/kami/maven/internal/delivery"
|
|
"github.com/kami/maven/internal/loop"
|
|
"github.com/kami/maven/internal/phraser"
|
|
"github.com/kami/maven/internal/phraser/eval"
|
|
)
|
|
|
|
// The persona checks, run before she speaks (Vikunja #399).
|
|
//
|
|
// RunChecks and RunTalkChecks only ever ran from the eval package, so
|
|
// everything the fixtures measured was offline knowledge: we could say "about
|
|
// one reply in three is broken" and still ship every one of them. This runs the
|
|
// cheap half of that on the live path, and replaces a failing message with the
|
|
// deterministic floor.
|
|
//
|
|
// Which checks: the unambiguous string tests only — feminine self-reference,
|
|
// how she addresses him, and a leaked-reasoning test. Not length, which is
|
|
// path-specific, and not ontopic, which compares against fragments the fixture
|
|
// supplies and runtime does not have. Not hisgender either — see guardSpoken.
|
|
//
|
|
// No retry. A retry doubles the latency on the exact turn that is already going
|
|
// badly, and on the nudge path the moment has passed.
|
|
//
|
|
// The known cost, written down because it is real: a wrongly flagged good reply
|
|
// is replaced by a flatter stub one. That is the right trade — a stub sentence
|
|
// is dull, a leaked reasoning trace is broken — but it means these checks can
|
|
// no longer be tuned for sensitivity alone.
|
|
|
|
// checkLeak — the name reported when the model's scaffolding reaches the text.
|
|
const checkLeak = "leak"
|
|
|
|
// leakPatterns — reasoning and protocol that belongs to the model, not to him.
|
|
// The resident model is a Thinking variant, so an unclosed reasoning block is
|
|
// the failure mode, not a hypothetical (Vikunja #398).
|
|
var leakPatterns = []*regexp.Regexp{
|
|
regexp.MustCompile(`(?i)<\s*/?\s*think`),
|
|
regexp.MustCompile(`(?i)thinking\s*(process|:)`),
|
|
regexp.MustCompile(`(?i)^\s*(assistant|user|system)\s*:`),
|
|
// Raw contract JSON: the parser already unwraps a good one, so a body that
|
|
// still carries the keys is one it could not read.
|
|
regexp.MustCompile(`"(response|mood|body|summary)"\s*:`),
|
|
// The persona block quoted back at him.
|
|
regexp.MustCompile(`(?i)(ты\s+—?\s*мэйвен|системный промпт|system prompt)`),
|
|
}
|
|
|
|
// checkPersonaLeak reports whether the model's own scaffolding is in the text.
|
|
func checkPersonaLeak(body string) (string, bool) {
|
|
for _, re := range leakPatterns {
|
|
if m := re.FindString(body); m != "" {
|
|
return "leaked " + strings.TrimSpace(m), false
|
|
}
|
|
}
|
|
return "", true
|
|
}
|
|
|
|
// personaRejects counts what the guard caught, by check name, so the real
|
|
// production rate is knowable rather than inferred from the fixture.
|
|
var personaRejects = struct {
|
|
mu sync.Mutex
|
|
by map[string]int
|
|
}{by: map[string]int{}}
|
|
|
|
func personaRejectCounts() map[string]int {
|
|
personaRejects.mu.Lock()
|
|
defer personaRejects.mu.Unlock()
|
|
out := make(map[string]int, len(personaRejects.by))
|
|
for k, v := range personaRejects.by {
|
|
out[k] = v
|
|
}
|
|
return out
|
|
}
|
|
|
|
// guardSpoken checks a phrased message. It returns the failed check and false
|
|
// when the message must not be said; path names the caller, for the log.
|
|
//
|
|
// An empty message passes: the caller already treats that as a failure and
|
|
// falls back on its own, and reporting it as a persona breach would put a
|
|
// misleading line in the count.
|
|
func guardSpoken(path, body string) (string, bool) {
|
|
if strings.TrimSpace(body) == "" {
|
|
return "", true
|
|
}
|
|
if detail, ok := checkPersonaLeak(body); !ok {
|
|
return rejectSpoken(path, checkLeak, detail, body), false
|
|
}
|
|
// Feminine and address only. HisGender is not run here: it reads a
|
|
// sentence-initial feminine verb with no pronoun — "записала, что ты выпил
|
|
// воды" — as a woman being addressed, when it is her own correct
|
|
// self-reference. Offline that is a point of score; on this path it would
|
|
// replace a good reply with a stub one on every fact she confirms.
|
|
for _, r := range []eval.Result{eval.Feminine(body), eval.Address(body)} {
|
|
if !r.Pass {
|
|
return rejectSpoken(path, r.Name, r.Detail, body), false
|
|
}
|
|
}
|
|
return "", true
|
|
}
|
|
|
|
// rejectSpoken logs what she nearly said and counts it. The whole text, not a
|
|
// prefix: the point of the log line is that the failure can be read back later
|
|
// and argued with.
|
|
func rejectSpoken(path, check, detail, body string) string {
|
|
personaRejects.mu.Lock()
|
|
personaRejects.by[check]++
|
|
personaRejects.mu.Unlock()
|
|
log.Printf("persona: %s rejected on %s (%s): %q", path, check, detail, body)
|
|
return check
|
|
}
|
|
|
|
// guardNudge checks a phrased nudge and falls back to the deterministic floor
|
|
// when it fails. The nudge path, unlike the reply path, cannot ask again: the
|
|
// tick has already decided she speaks, so the choice is the floor's wording or
|
|
// a broken sentence.
|
|
func guardNudge(pn delivery.PhrasedNudge, cand loop.Candidate) delivery.PhrasedNudge {
|
|
if _, ok := guardSpoken("nudge", pn.Body); ok {
|
|
return pn
|
|
}
|
|
stub, err := phraser.NewStub().PhraseNudge(context.Background(), cand)
|
|
if err != nil {
|
|
// The Stub is templates over the candidate and does not fail. If it
|
|
// somehow does, the model's text is still what the rule decided to
|
|
// say, and saying nothing is the worse outcome.
|
|
return pn
|
|
}
|
|
return stub
|
|
}
|