package main import ( "bytes" "context" "log" "os" "strings" "testing" "time" "github.com/kami/maven/internal/dialogue" "github.com/kami/maven/internal/memory" "github.com/kami/maven/internal/router" "github.com/kami/maven/internal/store" ) // Dialogue contract tests (V-563, child of V-558). // // Every other clarify test is single-shot: one ask, one answer, one assertion. // Three bugs of the same family shipped in two days that way — V-554 (a parked // question ate the three turns after it), V-557 (a confidently routed but // incomplete reminder parked nothing, so the answer was web-searched) and the // Rome case in V-558 (a side question was eaten as the time answer). None of // them is visible in one turn. The dialogue path is a state machine, so it can // be enumerated instead: whole traces, each with a per-turn expectation and an // expected END state — what was written to the store, and what is still parked. // // Two rules for the rows below. // // Where today's behaviour is correct, it is asserted. Where it is WRONG, the row // carries the CORRECT expectation and is skipped with the Vikunja id that will // unskip it. A weakened expectation would be worse than no row: it would pin the // bug as the contract. // // Everything runs on the offline floor — hash embedder, no llama-server, no // ONNX, StubDateTimeParser. That has one consequence worth knowing before // reading a fire time here: the stub reads "в 11:00" and "через час" and does // not read "на 9" or "на завтра", so a trace that needs those is noted where it // sits. // claim — which claimant consumed an utterance. Not asserted: it is derived from // the log lines the daemon already emits and printed on every failure, because // "the reply differed" does not distinguish a wrong claimant from wrong copy, // and that distinction is the whole point of V-558. type claim struct { utterance string steps []string } func (c claim) String() string { return c.utterance + " ⇒ " + strings.Join(c.steps, " → ") } // claimMarkers — log fragment to claimant name, in the order runTurn checks // them. The fragments are the daemon's own words (clarify.go, repair.go, // voice.go); a rename there shows up here as an "unclaimed" step rather than a // silent mislabel. var claimMarkers = []struct{ fragment, name string }{ {"parked question expired", "clarify:expired"}, {"is its own request", "clarify:stepped-aside"}, {"gave up on", "clarify:gave-up"}, {"did not fill", "clarify:re-ask"}, {"one gap filled", "clarify:ask-second-gap"}, {"asked about", "clarify:ask"}, {"repair —", "repair"}, {"route result: intent=", "route"}, } // claimsOf reads the turn's log output and names the claimants that touched it. func claimsOf(utterance, logged string) claim { c := claim{utterance: utterance} for _, line := range strings.Split(logged, "\n") { for _, m := range claimMarkers { if strings.Contains(line, m.fragment) { name := m.name if m.name == "route" { name = "route:" + intentInLine(line) } c.steps = append(c.steps, name) break } } } if len(c.steps) == 0 { c.steps = []string{"unclaimed"} } return c } func intentInLine(line string) string { _, rest, ok := strings.Cut(line, "intent=") if !ok { return "?" } intent, _, _ := strings.Cut(rest, " ") return intent } // parkedWant — the question that must be armed after a turn. Attempt matters: // a claimant that spends a retry on an utterance that was never an answer is // exactly the V-554 shape, and the count is the only place it shows. type parkedWant struct { slot dialogue.Slot attempt int // carries — a substring the parked utterance must still hold, so a re-park // that lost the answered subject fails here rather than three turns later. carries string } // turn — one utterance and everything that must be true right after it. type turn struct { say string // wait — the clock moves this far BEFORE the utterance. The only way to // reach the TTL without sleeping. wait time.Duration // question — the reply must be exactly this clarify question, worded for // this attempt. Zero slot ⇒ not checked. question dialogue.Slot attempt int contains []string notContain []string // noQuestion — the reply must not be any clarify question. Used where the // correct behaviour is known but her wording for it is not written yet: a // cancel must not be answered with another question, whatever it does say. noQuestion bool expired bool // the reply must open with the TTL notice // parked — what is armed after the turn. nil ⇒ nothing may be armed. parked *parkedWant } // endState — what the store holds once the trace is over. Counts and // substrings, not rows: a trace is about who claimed what, and a payload // substring is enough to catch a request landing under the wrong words. type endState struct { reminders []reminderWant factKeys []string notes int tasks []string } type reminderWant struct { payload string // substring of the stored payload fireAt string // "2006-01-02 15:04" in UTC, "" ⇒ not checked } // trace — a named conversation, its turns, and the end state. type trace struct { name string skip string // non-empty ⇒ t.Skip: today's behaviour is wrong, this names the fix turns []turn end endState } // newDialogueHandler — the offline floor with the real cascade and a movable // clock: newClarifyHandler's wiring (stub date parser, real fact parser, tool // matcher) plus the router newRoutingClarifyHandler builds, and the `now` // pointer so a turn can carry a wait. func newDialogueHandler(t *testing.T) (*reactiveHandler, *store.Store, *time.Time) { t.Helper() h, st, now := newClarifyHandler(t) h.router = buildRouter(router.NewHashEmbedder(1024), h.matcher, 0.55, nil) h.recall = recallWiring{embedder: router.NewHashEmbedder(1024), memStore: memory.NewInMemoryStore()} return h, st, now } // runTrace drives one trace through handleText and checks every turn, then the // end state. Every failure carries the decision trace so far, so a wrong // claimant reads differently from wrong copy. func runTrace(t *testing.T, tr trace) { t.Helper() // MAVEN_DIALOGUE_NO_SKIP=1 runs the rows that fail today. That is how // whoever lands V-560, V-561 or V-562 sees their row go green before // deleting its skip, and it is also the check that a skip is still earned: // a row that passes with the skip in place is a fix nobody noticed. if tr.skip != "" && os.Getenv("MAVEN_DIALOGUE_NO_SKIP") == "" { t.Skip(tr.skip) } ctx := context.Background() h, st, now := newDialogueHandler(t) const conversation = "web" id := dialogueIDFor(sourceText, conversation) var claims []claim fail := func(turnIdx int, format string, args ...any) { t.Helper() lines := make([]string, 0, len(claims)) for _, c := range claims { lines = append(lines, " "+c.String()) } t.Fatalf("turn %d: "+format+"\n who claimed what:\n%s", append([]any{turnIdx}, append(args, strings.Join(lines, "\n"))...)...) } for i, tn := range tr.turns { if tn.wait > 0 { *now = now.Add(tn.wait) } var logged bytes.Buffer prev := log.Writer() log.SetOutput(&logged) reply := h.handleText(ctx, conversation, tn.say) log.SetOutput(prev) claims = append(claims, claimsOf(tn.say, logged.String())) body := reply if tn.expired { if !isClarifyExpired(reply) { fail(i, "reply %q must open with the expiry notice", reply) } body = trimClarifyExpired(reply) // The notice is glued in front of this turn's reply, and both halves // have to survive: the words he just said are routed fresh, and // answering only "I let the old one go" drops them. if body == "" { fail(i, "the notice was the whole reply; the fresh words were never answered") } } else if isClarifyExpired(reply) { fail(i, "reply %q announced an expiry nothing asked for", reply) } if tn.question != "" { want, ok := clarifyQuestionFor(tn.question, tn.attempt) if !ok { fail(i, "no question exists for slot %s attempt %d", tn.question, tn.attempt) } if body != want { fail(i, "reply %q, want the %s question worded for attempt %d, %q", body, tn.question, tn.attempt, want) } } if tn.noQuestion && isAnyClarifyQuestion(body) { fail(i, "reply %q is another question; this turn is not something to ask about", body) } for _, want := range tn.contains { if !strings.Contains(body, want) { fail(i, "reply %q does not carry %q", body, want) } } for _, unwanted := range tn.notContain { if strings.Contains(body, unwanted) { fail(i, "reply %q carries %q and must not", body, unwanted) } } checkParked(t, fail, i, h.clarifyStore.Get(id, h.now()), tn.parked) } checkEnd(t, ctx, st, h, tr.end, claims) } // isAnyClarifyQuestion — is this reply one of her clarify questions, at any // attempt wording? Reads the templates rather than a list of its own. func isAnyClarifyQuestion(reply string) bool { for _, variants := range clarifyQuestionVariants { for _, v := range variants { if reply == v { return true } } } return false } func checkParked(t *testing.T, fail func(int, string, ...any), i int, got *dialogue.PendingQuestion, want *parkedWant) { t.Helper() if want == nil { if got != nil { fail(i, "a question about %v is still armed and nothing should be: %+v", got.Missing, got.Slots) } return } if got == nil { fail(i, "nothing is armed, want a question about %s (attempt %d)", want.slot, want.attempt) return } if len(got.Missing) != 1 || got.Missing[0] != want.slot { fail(i, "armed question is about %v, want %s", got.Missing, want.slot) } if got.Attempts != want.attempt { fail(i, "armed question is on attempt %d, want %d — a retry spent on something that was never an answer is the V-554 shape", got.Attempts, want.attempt) } if want.carries != "" && !strings.Contains(got.Utterance, want.carries) { fail(i, "the parked request no longer carries %q: %q", want.carries, got.Utterance) } } func checkEnd(t *testing.T, ctx context.Context, st *store.Store, h *reactiveHandler, want endState, claims []claim) { t.Helper() lines := make([]string, 0, len(claims)) for _, c := range claims { lines = append(lines, " "+c.String()) } trace := "\n who claimed what:\n" + strings.Join(lines, "\n") reminders, err := st.DueReminders(ctx, h.now().Add(14*24*time.Hour)) if err != nil { t.Fatalf("DueReminders: %v", err) } if len(reminders) != len(want.reminders) { t.Fatalf("end state: %d reminder(s), want %d: %+v%s", len(reminders), len(want.reminders), reminders, trace) } for i, w := range want.reminders { if !strings.Contains(reminders[i].Payload, w.payload) { t.Fatalf("end state: reminder %d payload %q does not carry %q%s", i, reminders[i].Payload, w.payload, trace) } if w.fireAt != "" { if got := reminders[i].FireTs.UTC().Format("2006-01-02 15:04"); got != w.fireAt { t.Fatalf("end state: reminder %d fires at %s, want %s%s", i, got, w.fireAt, trace) } } } facts, err := st.RecentFacts(ctx, 20) if err != nil { t.Fatalf("RecentFacts: %v", err) } if len(facts) != len(want.factKeys) { t.Fatalf("end state: %d fact(s), want %d: %+v%s", len(facts), len(want.factKeys), facts, trace) } for i, key := range want.factKeys { if facts[i].Key != key { t.Fatalf("end state: fact %d is %q, want %q%s", i, facts[i].Key, key, trace) } } notes, err := st.RecentNotes(ctx, 20) if err != nil { t.Fatalf("RecentNotes: %v", err) } if len(notes) != want.notes { t.Fatalf("end state: %d note(s), want %d%s", len(notes), want.notes, trace) } tasks, err := st.ListTasks(ctx, store.TaskOpen) if err != nil { t.Fatalf("ListTasks: %v", err) } if len(tasks) != len(want.tasks) { t.Fatalf("end state: %d open task(s), want %d: %+v%s", len(tasks), len(want.tasks), tasks, trace) } for i, text := range want.tasks { if !strings.Contains(tasks[i].Text, text) { t.Fatalf("end state: task %d is %q, want it to carry %q%s", i, tasks[i].Text, text, trace) } } } func TestDialogueTraces(t *testing.T) { for _, tr := range dialogueTraces() { tr := tr t.Run(tr.name, func(t *testing.T) { runTrace(t, tr) }) } } // dialogueTraces — the fixture. Order is the order the shapes were found, not a // dependency: each trace builds its own handler and store. func dialogueTraces() []trace { return []trace{ // The plain two-turn shape, and the one every other row is a deviation // from: she asks for the time, he gives it, the reminder lands with the // subject he said in the FIRST turn. { name: "reminder completed over two turns", turns: []turn{ {say: "напомни позвонить маме", question: dialogue.SlotTime, attempt: 1, parked: &parkedWant{slot: dialogue.SlotTime, attempt: 1, carries: "маме"}}, {say: "в 11:00", contains: []string{"11:00"}, notContain: []string{"?"}}, }, end: endState{reminders: []reminderWant{{payload: "позвонить маме", fireAt: "2026-07-31 11:00"}}}, }, // The same shape on the fact path, where the answer carries both halves // of what was missing — the key and the value — in one breath. { name: "fact completed over two turns", turns: []turn{ {say: "запиши", question: dialogue.SlotKey, attempt: 1, parked: &parkedWant{slot: dialogue.SlotKey, attempt: 1}}, {say: "пил воду", contains: []string{"water"}}, }, end: endState{factKeys: []string{"water"}}, }, // An answer past the TTL is a new request, not an answer (V-385). She // says the old one is gone and routes the words fresh. A bare time on // its own carries no request, so the fresh routing lands on the canned // reply — the point of the row is that NOTHING is created: a reminder // here would fire with the subject of a request she had already let go. { name: "answer arrives after the TTL", turns: []turn{ {say: "напомни позвонить маме", question: dialogue.SlotTime, attempt: 1, parked: &parkedWant{slot: dialogue.SlotTime, attempt: 1}}, {say: "в 11:00", wait: clarifyTTL + time.Second, expired: true}, }, end: endState{}, }, // Three questions is the budget, and running out is SPOKEN: a mute // give-up reads as "done" and he would wait for a reminder that was // never set. The wording changes with the attempt (V-457). { name: "three unclear answers then the give-up line", turns: []turn{ {say: "напомни позвонить маме", question: dialogue.SlotTime, attempt: 1, parked: &parkedWant{slot: dialogue.SlotTime, attempt: 1}}, {say: "ну не знаю", question: dialogue.SlotTime, attempt: 2, parked: &parkedWant{slot: dialogue.SlotTime, attempt: 2}}, {say: "ну не знаю", question: dialogue.SlotTime, attempt: 3, parked: &parkedWant{slot: dialogue.SlotTime, attempt: 3}}, {say: "ну не знаю", contains: []string{clarifyGaveUp}, noQuestion: true}, }, end: endState{}, }, // A correction points at the previous ACTED turn (repair.go): she redoes // it under the intent he names and says so out loud, because a // correction he cannot see is indistinguishable from one that was // dropped. The task she filed first stays filed — repair redoes, it does // not retract, and V-455 decided that deliberately. // // The corrected-to intent has to differ from the one she used, or repair // declines: teaching the classifier the label it already produced is // worse than doing nothing. { name: "correction of the previous turn", turns: []turn{ {say: "добавь в задачи купить молоко", contains: []string{"купить молоко"}}, {say: "нет, это был вопрос", contains: []string{"поняла, это вопрос"}}, }, end: endState{tasks: []string{"купить молоко"}}, }, // He walks away from his own request: a question is parked, the next // utterance is an unrelated request of its own, and nothing follows. // V-554's fix is what makes this row pass — the question steps aside // rather than scoring "добавь в задачи" as the time. The reminder is // dropped in silence and that is the decision: if he meant it he says it // again, and a question left armed eats the turn after next. { name: "abandoned flow: parked, then an unrelated request", turns: []turn{ {say: "напомни позвонить маме", question: dialogue.SlotTime, attempt: 1, parked: &parkedWant{slot: dialogue.SlotTime, attempt: 1}}, {say: "добавь в задачи купить молоко", contains: []string{"купить молоко"}}, {say: "спасибо"}, }, end: endState{tasks: []string{"купить молоко"}}, }, // ---- rows below carry the CORRECT expectation and fail today ---- // The owner's target transcript, V-561. He asks for a reminder, she asks // when, he asks something else entirely, and then comes back to her // question. On the box this created a reminder at 00:12 and never // answered Rome; on the offline floor the side question is recognised as // its own request and the flow is dropped instead, so the wrong reminder // is not made and the right one is not either. // // Both are the same defect: there is no suspend and resume. The correct // shape is the middle turn answered on its own and the parked question // still standing, on the same attempt — a side query is not a failed // answer and must not spend a retry. // // Unskipping this needs more than V-561. "на 9" and "на завтра" are not // read by StubDateTimeParser, which is what the offline floor runs, so // the row below it is the same shape in words the floor can parse and is // the one to watch first. { name: "the owner's transcript from V-561", skip: "V-561: a parked question is not suspended for a side query and never resumes", turns: []turn{ {say: "напомни позвонить маме", question: dialogue.SlotTime, attempt: 1, parked: &parkedWant{slot: dialogue.SlotTime, attempt: 1}}, {say: "какая сейчас погода в Риме?", parked: &parkedWant{slot: dialogue.SlotTime, attempt: 1, carries: "маме"}}, {say: "а, да, прости - на 9.", parked: &parkedWant{slot: dialogue.SlotTime, attempt: 1, carries: "маме"}}, {say: "на завтра."}, }, end: endState{reminders: []reminderWant{{payload: "позвонить маме", fireAt: "2026-08-01 09:00"}}}, }, // The same shape said in words StubDateTimeParser reads, so this row // turns green on V-561 alone. Same three claims: Rome is answered, the // question survives the side query on the same attempt, and the answer // after it completes the reminder he actually asked for. { name: "nested question: a parked question, then one of his own", skip: "V-561: a side query drops the parked question instead of suspending it", turns: []turn{ {say: "напомни позвонить маме", question: dialogue.SlotTime, attempt: 1, parked: &parkedWant{slot: dialogue.SlotTime, attempt: 1}}, {say: "какая сейчас погода в Риме?", parked: &parkedWant{slot: dialogue.SlotTime, attempt: 1, carries: "маме"}}, {say: "в 11:00", contains: []string{"11:00"}}, }, end: endState{reminders: []reminderWant{{payload: "позвонить маме", fireAt: "2026-07-31 11:00"}}}, }, // A cancel is one of the five turn roles V-560 names, and today it is // none of them: "неважно" fills no slot and carries no request of its // own, so it reads as a failed answer and spends a retry. Two turns // later she is still asking about a reminder he called off. // // The row asserts what is knowable — nothing armed, nothing written, and // not another question — rather than her wording for it, which is not // written yet and is not this task's to invent. { name: "cancel: a parked question, then never mind", skip: "V-560: a cancel is scored as a failed answer, not as a cancel", turns: []turn{ {say: "напомни позвонить маме", question: dialogue.SlotTime, attempt: 1, parked: &parkedWant{slot: dialogue.SlotTime, attempt: 1}}, {say: "неважно", noQuestion: true}, }, end: endState{}, }, // Order in runTurn is the whole arbitration (V-558), and this is what it // costs: the clarify answer is checked at step 3 and the repair marker at // step 4d, so while a question is parked no correction can be made. She // scores "нет, это была заметка" as a bad time answer and asks again. { name: "correction while a question is parked", skip: "V-560: clarify pre-empts the repair marker, so a correction cannot be spoken mid-flow", turns: []turn{ {say: "добавь в задачи купить молоко", contains: []string{"купить молоко"}}, {say: "напомни позвонить маме", question: dialogue.SlotTime, attempt: 1, parked: &parkedWant{slot: dialogue.SlotTime, attempt: 1}}, {say: "нет, это был вопрос", contains: []string{"поняла, это вопрос"}, parked: &parkedWant{slot: dialogue.SlotTime, attempt: 1, carries: "маме"}}, }, end: endState{tasks: []string{"купить молоко"}}, }, // A reminder said whole, in one breath, with the hour in it — and she // asks when. ReminderGrammar (stage0.go) builds its slots by hand and // never runs the extractor, so a stage-0 reminder carries no time // whatever the sentence says, and the clarify gate reads the gap as // real. It costs a turn on the commonest reminder shape there is. // // Hermetic despite the date parser: stage 0 calls no parser at all, so // this fails the same way with or without python dateparser installed. { name: "a reminder said whole is not asked about", skip: "V-562: a stage-0 decision never meets the extractor, so its slots are never validated", turns: []turn{ {say: "напомни в 11:00 позвонить маме", contains: []string{"11:00"}, noQuestion: true}, }, end: endState{reminders: []reminderWant{{payload: "позвонить маме", fireAt: "2026-07-31 11:00"}}}, }, // The same gap on the repair path. A correction redoes the request // through finishClarified, which goes straight to applyAction — it never // passes the clarify gate — so a redo that lands short answers with the // parse error V-557 removed from the routing path: "не поняла, на когда // напомнить." She should ask, exactly as she does for a fresh reminder // with no time. { name: "a correction that lands short asks rather than failing", skip: "V-562: finishClarified skips the clarify gate, so a repaired decision is never checked for gaps", turns: []turn{ {say: "добавь в задачи купить молоко", contains: []string{"купить молоко"}}, {say: "нет, это было напоминание", contains: []string{"поняла, это напоминание"}, parked: &parkedWant{slot: dialogue.SlotTime, attempt: 1}}, }, end: endState{tasks: []string{"купить молоко"}}, }, } }