Merge the persona floor guard (#263)
The three persona checks score what Variants() returns, and Variants() reads the JSON. The hardcoded Go floor strings were in no scored set, so the persona was unchecked precisely when the Go code rather than the model is doing the talking. Those floors are what speaks when the model is unreachable, and the CPT that would fix the persona in the model has not shipped. The floors live in nine files, not the four I named: acts.go holds the largest set at 35 lines and was not on my list. prompts.go, replier.go and llmphraser.go hold Russian written FOR the model, which must not be scored -- ruleTopics says 'он давно не пил воду', correct as prompt input and a CheckAddress failure on sight. TestGoFloorPersona reads the maps whole and calls the composing functions, so a new map entry is scored with no edit. TestGoFloorCoverage parses the package with go/ast and fails on any Russian literal that neither reached that corpus nor sits inside a declared prompt builder. The exemption list is of builders rather than strings, so the default for a literal added anywhere else is 'must be scored'. Named hole: a new literal that is a substring of an already-scored line passes silently. No existing floor violates the persona. The hand sweep was right; this makes it a guard. (V-621)
This commit is contained in:
@@ -0,0 +1,316 @@
|
||||
package phraser
|
||||
|
||||
// The persona guard for the Go floor strings.
|
||||
//
|
||||
// internal/phraser/eval/fallbacks_test.go already scores everything Variants()
|
||||
// returns — that is the JSON decks. What it cannot see is the floor UNDER those
|
||||
// decks: the hardFloor/ackFloor/queryFloor/actFloor/confirmFloor maps and the
|
||||
// literals in nudge_llm.go, which are what she says when the JSON is unusable or
|
||||
// when the model is unreachable. Those are exactly the moments the model is not
|
||||
// doing the talking, so leaving them unscored left the persona unchecked when it
|
||||
// was most load-bearing (Vikunja #621).
|
||||
//
|
||||
// Two tests here, and the second one is the point:
|
||||
//
|
||||
// - TestGoFloorPersona scores the floor corpus on the same checks.
|
||||
// - TestGoFloorCoverage walks the package source with go/ast and fails on any
|
||||
// Russian string literal that neither reached the corpus nor sits inside a
|
||||
// declaration declared prompt-side. A hand-written list of strings would rot
|
||||
// the first time somebody adds one; a hand-written list of PROMPT BUILDERS
|
||||
// does not, because the default for a new literal is "must be scored".
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"io/fs"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/kami/maven/internal/loop"
|
||||
"github.com/kami/maven/internal/phraser/eval"
|
||||
"github.com/kami/maven/internal/store"
|
||||
)
|
||||
|
||||
// personaChecks — the checks that apply to a floor line.
|
||||
//
|
||||
// The same three the non-goals section names (feminine self-reference, how she
|
||||
// addresses him, no pet names), plus lang: an English floor line is unusable out
|
||||
// loud. No hisgender, for the reason eval/fallbacks_test.go gives — her own
|
||||
// feminine verb near "тебе" is correct and that check reads it as addressing him
|
||||
// as a woman. No length, because a floor line composed from his own data has no
|
||||
// bounded length, and no ontopic/mood, which need a fixture case.
|
||||
var personaChecks = map[string]bool{
|
||||
eval.CheckLang: true,
|
||||
eval.CheckFeminine: true,
|
||||
eval.CheckAddress: true,
|
||||
eval.CheckCringe: true,
|
||||
}
|
||||
|
||||
var floorPlaceholderRE = regexp.MustCompile(`\{[a-z_]+\}`)
|
||||
|
||||
// formatVerbRE — the fmt verbs a floor line is composed with, so the coverage
|
||||
// test compares the Russian either side of them and not the verb.
|
||||
var formatVerbRE = regexp.MustCompile(`%[+\-# 0-9.]*[a-zA-Z]`)
|
||||
|
||||
// floorLine — one scored string and where it came from, so a failure names the
|
||||
// map or the function to go and edit.
|
||||
type floorLine struct {
|
||||
origin string
|
||||
text string
|
||||
}
|
||||
|
||||
// floorCorpus — every line the Go floor can produce. Maps are read whole, so a
|
||||
// new entry in one is scored without touching this file; the composing functions
|
||||
// are CALLED rather than scraped, so their glue text is scored in place.
|
||||
func floorCorpus() []floorLine {
|
||||
var out []floorLine
|
||||
add := func(origin, text string) {
|
||||
if strings.TrimSpace(text) != "" {
|
||||
out = append(out, floorLine{origin, text})
|
||||
}
|
||||
}
|
||||
for name, m := range map[string]map[string]string{
|
||||
"fallbacks.go hardFloor": hardFloor,
|
||||
"acks.go ackFloor": ackFloor,
|
||||
"query.go queryFloor": queryFloor,
|
||||
"acts.go actFloor": actFloor,
|
||||
"confirm.go confirmFloor": confirmFloor,
|
||||
"nudge_llm.go fallbackNudges": fallbackNudges,
|
||||
} {
|
||||
for key, text := range m {
|
||||
add(name+"["+key+"]", text)
|
||||
}
|
||||
}
|
||||
|
||||
// fallbackNudge composes three of its four arms in Go. Drive every rule name
|
||||
// the maps know, one it does not, and the down-services arm.
|
||||
rules := map[string]bool{"": true, "unknown_rule": true}
|
||||
for name := range fallbackNudges {
|
||||
rules[name] = true
|
||||
}
|
||||
for name := range ruleTopics {
|
||||
rules[name] = true
|
||||
}
|
||||
for name := range ruleKeywords {
|
||||
rules[name] = true
|
||||
}
|
||||
for name := range rules {
|
||||
c := loop.Candidate{}
|
||||
c.Rule.Name = name
|
||||
add(fmt.Sprintf("nudge_llm.go fallbackNudge(%q)", name), fallbackNudge(c))
|
||||
}
|
||||
// The keyword arm again, through a rule name shaped "family:keyword", which
|
||||
// is where ruleKeyword's second branch lives.
|
||||
c := loop.Candidate{}
|
||||
c.Rule.Name = "custom:зарядку"
|
||||
add("nudge_llm.go fallbackNudge(custom)", fallbackNudge(c))
|
||||
|
||||
// The keywords themselves. A rule that has both a keyword and a fallback
|
||||
// line never reaches the keyword arm, but the map is edited as one thing and
|
||||
// the next rule may have only the keyword, so score every value.
|
||||
for rule, kw := range ruleKeywords {
|
||||
add("nudge_llm.go ruleKeywords["+rule+"]", "Напоминаю: "+kw+".")
|
||||
}
|
||||
|
||||
// The down-services arm, which needs a service actually reading down.
|
||||
down := loop.Candidate{}
|
||||
down.Rule.Name = "service_down"
|
||||
down.State.Facts = map[string]store.Fact{
|
||||
loop.ServiceDownPrefix + "gitea": {
|
||||
Key: loop.ServiceDownPrefix + "gitea",
|
||||
Value: `"down"`,
|
||||
Source: loop.ServiceDownSource,
|
||||
Ts: time.Now(),
|
||||
},
|
||||
}
|
||||
add("nudge_llm.go fallbackNudge(down services)", fallbackNudge(down))
|
||||
|
||||
// The spoken duration words. Both functions are pure and bounded, so scoring
|
||||
// their whole range beats scraping the literals out of the switch.
|
||||
for m := 0; m <= 60*30; m += 7 {
|
||||
d := time.Duration(m) * time.Minute
|
||||
add("nudge_llm.go ruDur", ruDur(d))
|
||||
add("nudge_templates.go ruSinceWords", ruSinceWords(d))
|
||||
}
|
||||
for h := 0; h <= hoursSpoken; h++ {
|
||||
add("nudge_templates.go hourPlural", hourPlural(h))
|
||||
add("nudge_templates.go hourWord", hourWord(h))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestGoFloorPersona scores every line the Go floor can say.
|
||||
func TestGoFloorPersona(t *testing.T) {
|
||||
corpus := floorCorpus()
|
||||
if len(corpus) == 0 {
|
||||
t.Fatal("no floor lines — the corpus builder found nothing to score")
|
||||
}
|
||||
for _, line := range corpus {
|
||||
// A placeholder stands for his own words and carries no persona.
|
||||
body := floorPlaceholderRE.ReplaceAllString(line.text, "вода")
|
||||
for _, r := range eval.RunChecks(eval.Case{}, body, "neutral") {
|
||||
if personaChecks[r.Name] && !r.Pass {
|
||||
t.Errorf("%s: %q fails %s: %s", line.origin, line.text, r.Name, r.Detail)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// promptDecls — declarations whose Russian is written FOR the model, not for
|
||||
// him. They are excluded by name, not by string, so adding a line inside one of
|
||||
// them stays excluded and adding a line anywhere else fails the coverage test.
|
||||
//
|
||||
// Every name here is asserted to still exist, so a rename fails loudly instead
|
||||
// of silently widening the exemption.
|
||||
var promptDecls = map[string]string{
|
||||
"ReplySystemPrompt": "the system prompt for the reply model",
|
||||
"replyContext": "renders the decision FOR the model, never spoken",
|
||||
"ruleTopics": "situation descriptions fed to the nudge prompt",
|
||||
"ruleTopic": "same, plus the two prefixes it composes",
|
||||
"buildNudgePrompt": "the nudge prompt itself",
|
||||
"chatUserMessage": "the history block handed to the model",
|
||||
"PhraseReminder": "the reminder prompt; its reply is scored, its prompt is not",
|
||||
}
|
||||
|
||||
// promptFiles — files whose whole job is prompt text. Asserted to exist, same
|
||||
// reason as promptDecls.
|
||||
var promptFiles = map[string]string{
|
||||
"prompts.go": "every literal in it is a prompt",
|
||||
}
|
||||
|
||||
func hasCyrillic(s string) bool {
|
||||
for _, r := range s {
|
||||
if unicode.Is(unicode.Cyrillic, r) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// TestGoFloorCoverage — the guard that survives the next person.
|
||||
//
|
||||
// It reads the package source and requires every Russian string literal to be
|
||||
// one of two things: reachable in floorCorpus (so TestGoFloorPersona scored it),
|
||||
// or inside a declaration named above as prompt-side. There is no third answer
|
||||
// and no way to add a floor string that quietly gets neither.
|
||||
func TestGoFloorCoverage(t *testing.T) {
|
||||
var scored []string
|
||||
for _, line := range floorCorpus() {
|
||||
scored = append(scored, line.text)
|
||||
}
|
||||
// A literal is covered when every Russian piece of it shows up in something
|
||||
// the corpus scored. Pieces, not the whole string, because a format string
|
||||
// ("%d ч") and a concatenation fragment ("Не отвечает: ") only ever reach him
|
||||
// with the surrounding value filled in.
|
||||
covered := func(lit string) bool {
|
||||
for _, part := range formatVerbRE.Split(lit, -1) {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" || !hasCyrillic(part) {
|
||||
continue
|
||||
}
|
||||
found := false
|
||||
for _, s := range scored {
|
||||
if strings.Contains(s, part) {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
fset := token.NewFileSet()
|
||||
pkgs, err := parser.ParseDir(fset, ".", func(fi fs.FileInfo) bool {
|
||||
return !strings.HasSuffix(fi.Name(), "_test.go")
|
||||
}, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("parse package: %v", err)
|
||||
}
|
||||
pkg, ok := pkgs["phraser"]
|
||||
if !ok {
|
||||
t.Fatal("package phraser did not parse — the coverage guard cannot run")
|
||||
}
|
||||
|
||||
seenDecl := map[string]bool{}
|
||||
seenFile := map[string]bool{}
|
||||
for path, file := range pkg.Files {
|
||||
base := path[strings.LastIndexByte(path, '/')+1:]
|
||||
if _, exempt := promptFiles[base]; exempt {
|
||||
seenFile[base] = true
|
||||
continue
|
||||
}
|
||||
for _, decl := range file.Decls {
|
||||
names := declNames(decl)
|
||||
skip := false
|
||||
for _, n := range names {
|
||||
if _, ok := promptDecls[n]; ok {
|
||||
seenDecl[n] = true
|
||||
skip = true
|
||||
}
|
||||
}
|
||||
if skip {
|
||||
continue
|
||||
}
|
||||
ast.Inspect(decl, func(n ast.Node) bool {
|
||||
bl, ok := n.(*ast.BasicLit)
|
||||
if !ok || bl.Kind != token.STRING {
|
||||
return true
|
||||
}
|
||||
lit, err := strconv.Unquote(bl.Value)
|
||||
if err != nil || !hasCyrillic(lit) {
|
||||
return true
|
||||
}
|
||||
if !covered(lit) {
|
||||
t.Errorf("%s: Russian literal %q is spoken by nothing the persona guard scores.\n"+
|
||||
"Either reach it from floorCorpus in persona_floor_test.go, or — if it is written "+
|
||||
"for the model rather than for him — name its declaration in promptDecls.",
|
||||
fset.Position(bl.Pos()), lit)
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
}
|
||||
for name, why := range promptDecls {
|
||||
if !seenDecl[name] {
|
||||
t.Errorf("promptDecls names %q (%s) and no such declaration exists — "+
|
||||
"a rename left the exemption open", name, why)
|
||||
}
|
||||
}
|
||||
for base, why := range promptFiles {
|
||||
if !seenFile[base] {
|
||||
t.Errorf("promptFiles names %q (%s) and no such file exists", base, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// declNames — the names a top-level declaration binds, so a prompt-side var,
|
||||
// const or func can be matched whatever kind it is.
|
||||
func declNames(decl ast.Decl) []string {
|
||||
switch d := decl.(type) {
|
||||
case *ast.FuncDecl:
|
||||
return []string{d.Name.Name}
|
||||
case *ast.GenDecl:
|
||||
var out []string
|
||||
for _, spec := range d.Specs {
|
||||
switch s := spec.(type) {
|
||||
case *ast.ValueSpec:
|
||||
for _, n := range s.Names {
|
||||
out = append(out, n.Name)
|
||||
}
|
||||
case *ast.TypeSpec:
|
||||
out = append(out, s.Name.Name)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user