84a75274bf
First slice: the types a multi-turn trace is written in, and the claimant trace read out of the daemon's own log lines. No rows yet.
144 lines
5.1 KiB
Go
144 lines
5.1 KiB
Go
package main
|
|
|
|
import (
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/kami/maven/internal/dialogue"
|
|
)
|
|
|
|
// Dialogue contract tests (V-563, child of V-558).
|
|
//
|
|
// Every other clarify test is single-shot: one ask, one answer, one assertion.
|
|
// Three bugs of the same family shipped in two days that way — V-554 (a parked
|
|
// question ate the three turns after it), V-557 (a confidently routed but
|
|
// incomplete reminder parked nothing, so the answer was web-searched) and the
|
|
// Rome case in V-558 (a side question was eaten as the time answer). None of
|
|
// them is visible in one turn. The dialogue path is a state machine, so it can
|
|
// be enumerated instead: whole traces, each with a per-turn expectation and an
|
|
// expected END state — what was written to the store, and what is still parked.
|
|
//
|
|
// Two rules for the rows below.
|
|
//
|
|
// Where today's behaviour is correct, it is asserted. Where it is WRONG, the row
|
|
// carries the CORRECT expectation and is skipped with the Vikunja id that will
|
|
// unskip it. A weakened expectation would be worse than no row: it would pin the
|
|
// bug as the contract.
|
|
//
|
|
// Everything runs on the offline floor — hash embedder, no llama-server, no
|
|
// ONNX, StubDateTimeParser. That has one consequence worth knowing before
|
|
// reading a fire time here: the stub reads "в 11:00" and "через час" and does
|
|
// not read "на 9" or "на завтра", so a trace that needs those is noted where it
|
|
// sits.
|
|
|
|
// claim — which claimant consumed an utterance. Not asserted: it is derived from
|
|
// the log lines the daemon already emits and printed on every failure, because
|
|
// "the reply differed" does not distinguish a wrong claimant from wrong copy,
|
|
// and that distinction is the whole point of V-558.
|
|
type claim struct {
|
|
utterance string
|
|
steps []string
|
|
}
|
|
|
|
func (c claim) String() string { return c.utterance + " ⇒ " + strings.Join(c.steps, " → ") }
|
|
|
|
// claimMarkers — log fragment to claimant name, in the order runTurn checks
|
|
// them. The fragments are the daemon's own words (clarify.go, repair.go,
|
|
// voice.go); a rename there shows up here as an "unclaimed" step rather than a
|
|
// silent mislabel.
|
|
var claimMarkers = []struct{ fragment, name string }{
|
|
{"parked question expired", "clarify:expired"},
|
|
{"is its own request", "clarify:stepped-aside"},
|
|
{"gave up on", "clarify:gave-up"},
|
|
{"did not fill", "clarify:re-ask"},
|
|
{"one gap filled", "clarify:ask-second-gap"},
|
|
{"asked about", "clarify:ask"},
|
|
{"repair —", "repair"},
|
|
{"route result: intent=", "route"},
|
|
}
|
|
|
|
// claimsOf reads the turn's log output and names the claimants that touched it.
|
|
func claimsOf(utterance, logged string) claim {
|
|
c := claim{utterance: utterance}
|
|
for _, line := range strings.Split(logged, "\n") {
|
|
for _, m := range claimMarkers {
|
|
if strings.Contains(line, m.fragment) {
|
|
name := m.name
|
|
if m.name == "route" {
|
|
name = "route:" + intentInLine(line)
|
|
}
|
|
c.steps = append(c.steps, name)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
if len(c.steps) == 0 {
|
|
c.steps = []string{"unclaimed"}
|
|
}
|
|
return c
|
|
}
|
|
|
|
func intentInLine(line string) string {
|
|
_, rest, ok := strings.Cut(line, "intent=")
|
|
if !ok {
|
|
return "?"
|
|
}
|
|
intent, _, _ := strings.Cut(rest, " ")
|
|
return intent
|
|
}
|
|
|
|
// parkedWant — the question that must be armed after a turn. Attempt matters:
|
|
// a claimant that spends a retry on an utterance that was never an answer is
|
|
// exactly the V-554 shape, and the count is the only place it shows.
|
|
type parkedWant struct {
|
|
slot dialogue.Slot
|
|
attempt int
|
|
// carries — a substring the parked utterance must still hold, so a re-park
|
|
// that lost the answered subject fails here rather than three turns later.
|
|
carries string
|
|
}
|
|
|
|
// turn — one utterance and everything that must be true right after it.
|
|
type turn struct {
|
|
say string
|
|
// wait — the clock moves this far BEFORE the utterance. The only way to
|
|
// reach the TTL without sleeping.
|
|
wait time.Duration
|
|
// question — the reply must be exactly this clarify question, worded for
|
|
// this attempt. Zero slot ⇒ not checked.
|
|
question dialogue.Slot
|
|
attempt int
|
|
contains []string
|
|
notContain []string
|
|
// noQuestion — the reply must not be any clarify question. Used where the
|
|
// correct behaviour is known but her wording for it is not written yet: a
|
|
// cancel must not be answered with another question, whatever it does say.
|
|
noQuestion bool
|
|
expired bool // the reply must open with the TTL notice
|
|
// parked — what is armed after the turn. nil ⇒ nothing may be armed.
|
|
parked *parkedWant
|
|
}
|
|
|
|
// endState — what the store holds once the trace is over. Counts and
|
|
// substrings, not rows: a trace is about who claimed what, and a payload
|
|
// substring is enough to catch a request landing under the wrong words.
|
|
type endState struct {
|
|
reminders []reminderWant
|
|
factKeys []string
|
|
notes int
|
|
tasks []string
|
|
}
|
|
|
|
type reminderWant struct {
|
|
payload string // substring of the stored payload
|
|
fireAt string // "2006-01-02 15:04" in UTC, "" ⇒ not checked
|
|
}
|
|
|
|
// trace — a named conversation, its turns, and the end state.
|
|
type trace struct {
|
|
name string
|
|
skip string // non-empty ⇒ t.Skip: today's behaviour is wrong, this names the fix
|
|
turns []turn
|
|
end endState
|
|
}
|