Files
Maven/internal/phraser/nudge_templates_test.go
T
kami 6b67e6f3c2 Word nudges from templates by default, model optional
DEPRECATION, flagged not asked: LLM-phrased nudges are no longer the default.
LLMPhraser.PhraseNudge now returns a hand-written Russian template. The model
still phrases chat, queries and reminders — only nudges moved.

Why: measured over many runs, Qwen3.5-0.8B wrote formal "вы" and plural
imperatives, used masculine self-reference, and invented facts and units
(90-95 seconds to boil an egg). A nudge is five words of known content, so
generation buys nothing and risks the persona every time. Templates score
15/15 on the nudge fixture, the model 11-13/15.

Nothing is deleted: the prompt, the fallbacks and the whole LLM nudge path
stay. Set phraser.llm_nudges = true in deploy/mavend.json to get them back.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ
2026-07-31 18:21:52 +04:00

203 lines
5.6 KiB
Go

package phraser
import (
"context"
"math/rand"
"strings"
"testing"
"time"
"github.com/kami/maven/internal/loop"
"github.com/kami/maven/internal/store"
)
// cand builds a candidate the way a tick would.
func cand(rule string, sinceMin int, factKey string) loop.Candidate {
now := time.Date(2026, 7, 31, 21, 40, 0, 0, time.UTC)
st := loop.State{Now: now, Facts: map[string]store.Fact{}}
if sinceMin > 0 || factKey != "" {
key := rule
if factKey != "" {
key = factKey
}
st.Facts[rule] = store.Fact{Key: key, Ts: now.Add(-time.Duration(sinceMin) * time.Minute)}
}
return loop.Candidate{Rule: loop.Rule{Name: rule, Severity: loop.Sev1}, Severity: loop.Sev1, State: st}
}
func newTestTemplates(t *testing.T, seed int64) *NudgeTemplates {
t.Helper()
nt, err := NewNudgeTemplates(rand.NewSource(seed))
if err != nil {
t.Fatalf("NewNudgeTemplates: %v", err)
}
return nt
}
func TestNudgeTemplatesLoad(t *testing.T) {
nt := newTestTemplates(t, 1)
for _, rule := range []string{"water", "meal", "break", "service_down", "netdata_critical", "routine", "morning", "default"} {
set, ok := nt.file.Rules[rule]
if !ok {
t.Errorf("no templates for %q", rule)
continue
}
if len(set.Variants) < 5 {
t.Errorf("%s: only %d variants", rule, len(set.Variants))
}
// Every rule needs one variant that needs no value, or a candidate
// without context has nothing to say. routine and morning are exempt:
// they always carry a name and must always say it.
plain := 0
seen := map[string]bool{}
for _, v := range set.Variants {
if !placeholderRE.MatchString(v) {
plain++
}
if seen[v] {
t.Errorf("%s: duplicate variant %q", rule, v)
}
seen[v] = true
}
if plain == 0 && rule != "routine" && rule != "morning" {
t.Errorf("%s: every variant needs a placeholder value", rule)
}
}
}
// The whole point of the picker: never the same words twice in a row.
func TestNudgeNoImmediateRepeat(t *testing.T) {
nt := newTestTemplates(t, 7)
prev := ""
for i := 0; i < 200; i++ {
body, _ := nt.Nudge(cand("water", 200, ""))
if body == prev {
t.Fatalf("repeat at %d: %q", i, body)
}
prev = body
}
}
// Same seed, same sequence — otherwise the fixture score would drift run to run.
func TestNudgeDeterministicWithSeed(t *testing.T) {
var runs [2][]string
for r := range runs {
nt := newTestTemplates(t, 42)
for i := 0; i < 20; i++ {
body, _ := nt.Nudge(cand("break", 100, ""))
runs[r] = append(runs[r], body)
}
}
for i := range runs[0] {
if runs[0][i] != runs[1][i] {
t.Fatalf("run %d differs: %q vs %q", i, runs[0][i], runs[1][i])
}
}
}
// A variant is only used when its value exists, and nothing half-filled ships.
func TestNudgeNoLeftoverPlaceholders(t *testing.T) {
nt := newTestTemplates(t, 3)
cases := []loop.Candidate{
cand("water", 0, ""), // no duration
cand("water", 30, ""), // under an hour
cand("water", 200, ""), // hours
cand("service_down", 3, "vaultwarden"),
cand("service_down", 3, ""), // no service name
cand("routine:таблетки", 0, ""),
cand("morning:утро", 0, ""),
cand("unknown_rule", 0, ""),
}
for _, c := range cases {
for i := 0; i < 40; i++ {
body, mood := nt.Nudge(c)
if body == "" {
t.Fatalf("%s: empty body", c.Rule.Name)
}
if strings.ContainsAny(body, "{}%") {
t.Fatalf("%s: unfilled template %q", c.Rule.Name, body)
}
if mood != "neutral" {
t.Fatalf("%s: mood %q", c.Rule.Name, mood)
}
}
}
}
// The routine name must actually land in the text.
func TestNudgeSubstitutesWhat(t *testing.T) {
nt := newTestTemplates(t, 11)
for i := 0; i < 40; i++ {
body, _ := nt.Nudge(cand("routine:таблетки", 0, ""))
if !strings.Contains(strings.ToLower(body), "таблетки") {
t.Fatalf("routine text lost the name: %q", body)
}
}
}
func TestRuSinceWords(t *testing.T) {
cases := []struct {
min int
want string
}{
{30, ""},
{60, "час"},
{95, "полтора часа"},
{150, "два с половиной часа"},
{190, "три часа"},
{240, "четыре часа"},
{430, "семь часов"},
{660, "одиннадцать часов"},
{60 * 30, "больше суток"},
}
for _, c := range cases {
got := ruSinceWords(time.Duration(c.min) * time.Minute)
if got != c.want {
t.Errorf("%d min: got %q want %q", c.min, got, c.want)
}
}
}
// Templates are the default: a nudge must not reach the model at all.
func TestLLMPhraserUsesTemplatesByDefault(t *testing.T) {
spy := newGrammarSpy(t)
p := NewLLMPhraserAt(spy.srv.URL, Config{})
pn, err := p.PhraseNudge(context.Background(), cand("water", 200, ""))
if err != nil {
t.Fatalf("PhraseNudge: %v", err)
}
if len(spy.grammars) != 0 {
t.Errorf("nudge hit the model %d times, want 0", len(spy.grammars))
}
if !strings.Contains(strings.ToLower(pn.Body), "вод") {
t.Errorf("nudge is not the water template: %q", pn.Body)
}
}
// ...and the flag brings the model back.
func TestLLMNudgesFlagRestoresTheModel(t *testing.T) {
spy := newGrammarSpy(t)
p := NewLLMPhraserAt(spy.srv.URL, Config{LLMNudges: true})
pn, err := p.PhraseNudge(context.Background(), cand("water", 200, ""))
if err != nil {
t.Fatalf("PhraseNudge: %v", err)
}
if len(spy.grammars) != 1 {
t.Fatalf("nudge hit the model %d times, want 1", len(spy.grammars))
}
if pn.Body != "ага" {
t.Errorf("body = %q, want the model's reply", pn.Body)
}
}
func TestNudgeTemplatesPhraseNudge(t *testing.T) {
nt := newTestTemplates(t, 5)
pn, err := nt.PhraseNudge(context.Background(), cand("water", 200, ""))
if err != nil {
t.Fatalf("PhraseNudge: %v", err)
}
if pn.Body == "" || pn.Summary != pn.Body || pn.Mood != "neutral" {
t.Fatalf("bad nudge: %+v", pn)
}
}