From 35018226ef643c08206512725c1314fcf754cf12 Mon Sep 17 00:00:00 2001 From: claude Date: Tue, 4 Aug 2026 00:33:41 +0400 Subject: [PATCH] eval: score the reply path, the fourth phrasing path (V-396) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nine reply cases and a fourth column in the talk report. The reply path is a separate object from the phraser in the daemon, so Pair joins a Talker and a Confirmer for a run that covers everything Maven says. Cases carry intent/key/value because the replier is phrased from the decision the router resolved, not from the raw utterance. Three of them are baits the other paths cannot produce: a masculine verb about himself that she must not copy onto herself, a polite plural input that must still come back на ты, and an unresolved note that invites a question a confirmation is not allowed to ask. Not scored against a model here — this box has no llama-server, and the baseline test is opt-in on MAVEN_LLM_URL. --- internal/phraser/eval/talk.go | 51 ++++++++++++++++-- internal/phraser/eval/talk_test.go | 19 ++++++- internal/phraser/eval/talk_v1.json | 84 ++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 6 deletions(-) diff --git a/internal/phraser/eval/talk.go b/internal/phraser/eval/talk.go index 2d071a7..6d239f8 100644 --- a/internal/phraser/eval/talk.go +++ b/internal/phraser/eval/talk.go @@ -26,20 +26,22 @@ import ( "time" "github.com/kami/maven/internal/dialogue" + "github.com/kami/maven/internal/router" ) //go:embed talk_v1.json var talkFixtureJSON []byte -// The three phrasing paths under test. Values match the fixture's "path" field. +// The phrasing paths under test. Values match the fixture's "path" field. const ( PathChat = "chat" // PhraseChat PathQuery = "query" // PhraseQuery with notes PathKnowledge = "knowledge" // PhraseQuery with no notes + PathReply = "reply" // PhraseReply, the reactive confirmation ) // TalkPaths — report order. -var TalkPaths = []string{PathChat, PathQuery, PathKnowledge} +var TalkPaths = []string{PathChat, PathQuery, PathKnowledge, PathReply} // TalkCheckNames — the checks that apply to a free-form reply, in report order. // Deliberately a subset of CheckNames: length, mood and "no questions" are nudge @@ -58,12 +60,19 @@ var TalkCheckNames = []string{ // WantAny is the on-topic contract: at least one lowercased fragment must appear // in the reply. Fragments are stems ("пароль" → "парол") so declension does not // defeat them. +// +// Intent, Key and Value carry the reply path's decision: that path is phrased +// from what the router already resolved, not from the raw utterance. Utterance +// stays filled anyway, because it is what a human reads in the report. type TalkCase struct { ID string `json:"id"` Path string `json:"path"` Utterance string `json:"utterance"` History []string `json:"history,omitempty"` Notes []string `json:"notes,omitempty"` + Intent string `json:"intent,omitempty"` + Key string `json:"key,omitempty"` + Value string `json:"value,omitempty"` WantAny []string `json:"want_any"` Tags []string `json:"tags,omitempty"` Note string `json:"note,omitempty"` @@ -92,13 +101,27 @@ func LoadTalk() (TalkFixture, error) { return f, nil } -// Talker — the two methods a conversational path must have to be scorable. -// *phraser.LLMPhraser satisfies it; same trick as Nudger. +// Talker — the methods a conversational path must have to be scorable. +// *phraser.LLMPhraser satisfies the first two; *phraser.Replier satisfies the +// third, so a run that scores all four paths passes a Pair. type Talker interface { PhraseChat(ctx context.Context, utterance string, history []dialogue.Turn) (string, error) PhraseQuery(ctx context.Context, utterance string, notes []string) (string, error) } +// Confirmer — the reply path. *phraser.Replier satisfies it. +type Confirmer interface { + PhraseReply(ctx context.Context, d router.Decision) (string, error) +} + +// Pair joins the two objects the daemon wires separately — the phraser and the +// replier — so one ScoreTalk call covers every path Maven speaks through. A bare +// Talker still works; its reply cases score as errors, which is honest. +type Pair struct { + Talker + Confirmer +} + // TalkOutcome — one scored case. type TalkOutcome struct { Case TalkCase @@ -194,10 +217,30 @@ func (c TalkCase) run(ctx context.Context, t Talker) (string, error) { return t.PhraseQuery(ctx, c.Utterance, c.Notes) case PathKnowledge: return t.PhraseQuery(ctx, c.Utterance, nil) + case PathReply: + conf, ok := t.(Confirmer) + if !ok { + return "", fmt.Errorf("target cannot phrase replies — pass a Pair") + } + return conf.PhraseReply(ctx, c.decision()) } return "", fmt.Errorf("unknown path %q", c.Path) } +// decision rebuilds what the router would have handed the replier. Text is the +// utterance for a note or a reminder, which is what the router puts there. +func (c TalkCase) decision() router.Decision { + return router.Decision{ + Intent: router.Intent(c.Intent), + Slots: router.Slots{ + Key: c.Key, + Value: c.Value, + Text: c.Utterance, + HasKey: c.Key != "", + }, + } +} + func (c TalkCase) turns() []dialogue.Turn { turns := make([]dialogue.Turn, 0, len(c.History)) for _, h := range c.History { diff --git a/internal/phraser/eval/talk_test.go b/internal/phraser/eval/talk_test.go index dcff579..7b993c9 100644 --- a/internal/phraser/eval/talk_test.go +++ b/internal/phraser/eval/talk_test.go @@ -11,6 +11,7 @@ import ( "github.com/kami/maven/internal/llm" "github.com/kami/maven/internal/persona" "github.com/kami/maven/internal/phraser" + "github.com/kami/maven/internal/router" ) // perPathMinimum — the resolution floor. A per-path score built on a handful of @@ -36,6 +37,10 @@ func TestTalkFixture(t *testing.T) { switch c.Path { case PathChat, PathQuery, PathKnowledge: + case PathReply: + if c.Intent == "" { + t.Errorf("%s: reply case has no intent — the replier is phrased from the decision", c.ID) + } default: t.Errorf("%s: unknown path %q", c.ID, c.Path) } @@ -69,10 +74,15 @@ type fakeTalker struct{ reply string } func (f fakeTalker) PhraseChat(context.Context, string, []dialogue.Turn) (string, error) { return f.reply, nil } + func (f fakeTalker) PhraseQuery(context.Context, string, []string) (string, error) { return f.reply, nil } +func (f fakeTalker) PhraseReply(context.Context, router.Decision) (string, error) { + return f.reply, nil +} + // TestScoreTalkCounts — a reply that fails on purpose must be counted on every // path, so a real run cannot report a hidden zero. func TestScoreTalkCounts(t *testing.T) { @@ -104,7 +114,7 @@ func TestScoreTalkCounts(t *testing.T) { } } -// TestLLMTalkBaseline — the resident model on the three conversational paths. +// TestLLMTalkBaseline — the resident model on all four phrasing paths. // Opt-in exactly like TestLLMPhrasingBaseline: CI has no model and a run costs // minutes on the CPU target. // @@ -148,7 +158,12 @@ func TestLLMTalkBaseline(t *testing.T) { } t.Logf("scoring model %s at %s", model, base) - rep, err := ScoreTalk(ctx, "llm ("+model+", built-in persona)", p, f) + // The reply path is a separate object in the daemon too: the phraser owns its + // own llama-server, the replier is handed an llm.Client. Pair scores both. + block := func() string { return persona.Facts{}.Block(time.Now()) } + target := Pair{Talker: p, Confirmer: phraser.NewReplier(llm.New(base, cfg.Timeout), block)} + + rep, err := ScoreTalk(ctx, "llm ("+model+", built-in persona)", target, f) if err != nil { t.Fatalf("ScoreTalk: %v", err) } diff --git a/internal/phraser/eval/talk_v1.json b/internal/phraser/eval/talk_v1.json index 64b428b..389cf3f 100644 --- a/internal/phraser/eval/talk_v1.json +++ b/internal/phraser/eval/talk_v1.json @@ -222,6 +222,90 @@ "utterance": "почему гром слышно позже молнии?", "want_any": ["звук", "све", "быстр", "гром", "молни"], "tags": ["general"] + }, + { + "id": "reply-fact-coffee", + "path": "reply", + "intent": "fact", + "key": "кофе", + "value": "закончился", + "utterance": "кофе закончился", + "want_any": ["коф"], + "tags": ["fact"], + "note": "The plainest confirmation there is, and the sentence he hears most often." + }, + { + "id": "reply-fact-weight", + "path": "reply", + "intent": "fact", + "key": "вес", + "value": "82", + "utterance": "мой вес 82", + "want_any": ["вес", "82"], + "tags": ["fact", "number"], + "note": "A number must survive into the confirmation; a paraphrase that drops it is useless." + }, + { + "id": "reply-fact-pill", + "path": "reply", + "intent": "fact", + "key": "таблетки", + "value": "выпил", + "utterance": "таблетки выпил", + "want_any": ["таблетк"], + "tags": ["fact", "feminine"], + "note": "He says 'выпил', masculine and about himself. She must not copy the form onto herself." + }, + { + "id": "reply-note-router", + "path": "reply", + "intent": "note", + "utterance": "роутер перезагружается сам по ночам", + "want_any": ["роутер"], + "tags": ["note"] + }, + { + "id": "reply-note-long", + "path": "reply", + "intent": "note", + "utterance": "если диск снова отвалится, посмотреть кабель, а не контроллер, в прошлый раз был кабель", + "want_any": ["диск", "кабел"], + "tags": ["note", "length"], + "note": "A long note baits a long confirmation. One sentence is the contract." + }, + { + "id": "reply-reminder-evening", + "path": "reply", + "intent": "reminder", + "utterance": "напомни вечером полить цветы", + "want_any": ["цвет", "полит", "вечер"], + "tags": ["reminder"] + }, + { + "id": "reply-reminder-tomorrow", + "path": "reply", + "intent": "reminder", + "utterance": "напомни завтра позвонить в поликлинику", + "want_any": ["поликлиник", "позвон", "звон"], + "tags": ["reminder"] + }, + { + "id": "reply-formality-bait", + "path": "reply", + "intent": "note", + "utterance": "запишите пожалуйста что счётчики я сдал", + "want_any": ["счётчик", "счетчик"], + "tags": ["note", "persona-bait", "address"], + "note": "Polite plural in the input. The confirmation must still be на ты." + }, + { + "id": "reply-question-bait", + "path": "reply", + "intent": "note", + "utterance": "надо купить фильтр для воды, не помню какой", + "want_any": ["фильтр"], + "tags": ["note", "no-question"], + "note": "An unresolved note invites her to ask which filter. A confirmation does not ask." } ] }