diff --git a/Makefile b/Makefile index ec8aaf3..1486a70 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ PIPER_BIN := $(shell pwd)/deps/piper/piper PIPER_MODEL := $(shell pwd)/models/tts/ru_RU-irina-medium.onnx PIPER_ESPEAK := $(shell pwd)/deps/piper/espeak-ng-data -.PHONY: stt-fixtures test-stt-golden all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test fmt-check vet run-stt run-tts run-web download-embedder deps-go eval-router eval-recall eval-phrasing eval-models +.PHONY: simulate stt-fixtures test-stt-golden all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test fmt-check vet run-stt run-tts run-web download-embedder deps-go eval-router eval-recall eval-phrasing eval-models all: build @@ -91,6 +91,14 @@ vet: CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \ $(GO) vet ./internal/... ./cmd/... +# simulate — replay every scripted day under cmd/mavend/testdata/scenarios +# through the real router, store, tick loop and intake journal, on a fake clock +# (Vikunja #284). Verbose so the transcript of each scenario lands in the +# terminal. Also runs as part of `make test`; this target is for reading it. +simulate: + CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \ + $(GO) test -v -count=1 -run TestSimulator ./cmd/mavend/ + test: fmt-check vet CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \ $(GO) test -race -coverprofile=coverage.out ./internal/... ./cmd/... diff --git a/cmd/mavend/simulator_test.go b/cmd/mavend/simulator_test.go new file mode 100644 index 0000000..df00c2f --- /dev/null +++ b/cmd/mavend/simulator_test.go @@ -0,0 +1,798 @@ +// mavend/simulator_test.go — the replayable full-system simulator +// (Vikunja #284, 20-07-2026-BACKLOG.md item 7). +// +// # What it is +// +// A scripted day, replayed through the real mavend code paths, with every +// boundary faked and the clock under the scenario's control. A scenario is a +// JSON file in testdata/scenarios; the harness reads it, builds a world, walks +// the steps in order, and asserts on what actually happened: +// +// what Maven SAID — the reply text of every utterance +// what was SENT — every delivery.Sendable the dispatcher emitted +// what ARRIVED — the unified intake journal from #283 +// what TOOLS were called — the recorded requests against fake Praxis/Nexis/Hexis +// what did NOT happen — expect_no_send / expect_no_call, first-class +// +// The last one is the point. Maven's hard constraints are mostly negative — +// not a nag, not autonomous, nothing executed without confirmation — and a +// harness that can only assert on things that happened cannot test any of +// them. "Nothing was sent" is an assertion here, not an absence of one. +// +// # Determinism +// +// No time.Now() runs inside a replay. The scenario names a start instant, each +// step names a wall-clock offset from it, and the harness advances a fakeClock +// to that offset before running the step. Every clock reader in the world — +// the handler's `now`, the tick loop's `tick(ctx, now)`, the intake journal's +// publish stamp — is wired to that clock. Two runs of the same file produce +// the same transcript, and a scenario about 08:35 does not behave differently +// at 03:00 in CI. +// +// The tick is driven by the scenario, not by a ticker: tick() already takes +// `now` as an argument, so the only thing the daemon's ticker contributed was +// wall-clock timing, which is exactly what a replay must not have. +// +// # Why this shape and not a binary +// +// Vikunja #288 (golden-audio STT) deferred its tier-2 "audio → STT → router → +// phraser" scenarios to this task, and asked that they reuse a fixture format +// rather than inventing a third. A scenario here can name a WAV from +// cmd/mavsttd/testdata and the harness will feed it through the STT seam. As a +// test it runs under `make test` on every change, which a separate binary +// would not. +// +// # Production is untouched +// +// Every file this task adds is a _test.go file or testdata. There is no +// simulator in the daemon, no flag, no config key, and no code path that +// checks whether a simulation is running. The seams it uses — stt.Transcriber, +// tts.Synthesizer, router.Completer, delivery.Sink, ipc.CoreAPI, the +// event.Bus from #283 — all already existed for the production wiring. +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/kami/maven/internal/audio" + "github.com/kami/maven/internal/config" + "github.com/kami/maven/internal/delivery" + "github.com/kami/maven/internal/dialogue" + "github.com/kami/maven/internal/event" + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/llm" + "github.com/kami/maven/internal/loop" + "github.com/kami/maven/internal/phraser" + "github.com/kami/maven/internal/router" + "github.com/kami/maven/internal/store" + "github.com/kami/maven/internal/tool" + "github.com/kami/maven/internal/voice" +) + +// --------------------------------------------------------------------------- +// Scenario format +// --------------------------------------------------------------------------- + +// scenario — one scripted day. schema_version matches the convention already +// set by testdata/system_safety_scenarios.json. +type scenario struct { + SchemaVersion int `json:"schema_version"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + + // Start — the instant the day begins, RFC3339. Every step offset is + // relative to it, and nothing in the run reads a real clock. + Start string `json:"start"` + + // Script — what the resident model answers. The world has no llama-server; + // see scriptedLLM for how an entry is chosen. + Script []scriptEntry `json:"script,omitempty"` + + // Praxis / Nexus / Hexis — canned bodies for the ecosystem fakes. Absent ⇒ + // that service is not wired at all, which is the default box. + Praxis string `json:"praxis_attention,omitempty"` + Nexus string `json:"nexus_resolve,omitempty"` + Hexis string `json:"hexis_capabilities,omitempty"` + + Steps []step `json:"steps"` +} + +// scriptEntry — one canned model answer. Match is a substring of the user +// message; the first entry whose Match is contained in it wins, and an entry +// with an empty Match is the catch-all. +// +// Route and Reply are separate because the same model serves both contracts +// (CLAUDE.md, "LLM output contract"): a grammar-constrained call is a routing +// call and gets Route, an unconstrained one is a phrasing call and gets Reply. +type scriptEntry struct { + Match string `json:"match"` + Route string `json:"route,omitempty"` + Reply string `json:"reply,omitempty"` +} + +// step — one scripted moment. At is "HH:MM" or "HH:MM:SS", interpreted in the +// start instant's location; the clock is advanced to it before the step runs. +// +// A step does exactly one thing (say / audio / signal / fact / tick / arrive) +// and then asserts. Assertions are evaluated against everything recorded since +// the run began, except expect_no_send and expect_no_call, which are scoped to +// this step — "nothing was sent because of THIS" is the useful question. +type step struct { + At string `json:"at"` + Note string `json:"note,omitempty"` + + // --- stimuli (at most one per step) --- + + // Say — an utterance, as text, through the same runTurn the IPC chat path + // uses. + Say string `json:"say,omitempty"` + + // Audio — a WAV under cmd/mavsttd/testdata, fed through the STT seam. This + // is #288's deferred tier 2. The harness uses the deterministic stt stub + // unless a real transcriber is available, so the assertion a scenario can + // make about an audio step is about the PIPELINE, not about whisper's + // accuracy — that is what cmd/mavsttd/golden_test.go is for. + Audio string `json:"audio,omitempty"` + + // Signal — a presence/world fact arriving from a poller or /api/signal. + Signal *signalStep `json:"signal,omitempty"` + + // Arrive — an intake write from a module: an ambient notification, a feed + // item, a mail candidate. Goes through the same decorated ipc.CoreAPI the + // daemon gives those callers, so it lands in the journal exactly as it + // would in production. + Arrive *arriveStep `json:"arrive,omitempty"` + + // Tick — run one iteration of the proactive loop at this instant. + Tick bool `json:"tick,omitempty"` + + // Fault — make every ecosystem fake answer with this HTTP status from now + // on. The degraded-mode lever; ClearFault puts them back. + Fault int `json:"fault,omitempty"` + ClearFault bool `json:"clear_fault,omitempty"` + + // --- assertions --- + + ExpectReply []string `json:"expect_reply_contains,omitempty"` + ExpectNotReply []string `json:"expect_reply_lacks,omitempty"` + ExpectSent []string `json:"expect_sent_contains,omitempty"` + ExpectNoSend bool `json:"expect_no_send,omitempty"` + ExpectCalled []string `json:"expect_called,omitempty"` + ExpectNotCalled []string `json:"expect_not_called,omitempty"` + ExpectEvents []string `json:"expect_events,omitempty"` + ExpectNoEvents bool `json:"expect_no_events,omitempty"` +} + +type signalStep struct { + Key string `json:"key"` + Value string `json:"value"` + Source string `json:"source"` + Kind string `json:"kind,omitempty"` +} + +type arriveStep struct { + // Note / Fact / Task — exactly one. Each mirrors the intake seam its real + // caller uses. + Note *arriveNote `json:"note,omitempty"` + Fact *signalStep `json:"fact,omitempty"` + Task *arriveTask `json:"task,omitempty"` + AsOf string `json:"as_of,omitempty"` // "HH:MM" — OccurredAt, when it differs from the step time + Source string `json:"source"` +} + +type arriveNote struct { + Text string `json:"text"` +} + +type arriveTask struct { + Text string `json:"text"` + Evidence string `json:"evidence,omitempty"` + Status string `json:"status,omitempty"` +} + +// --------------------------------------------------------------------------- +// The world +// --------------------------------------------------------------------------- + +// simWorld — every faked boundary plus the real components between them. +type simWorld struct { + t *testing.T + clock *fakeClock + loc *time.Location + start time.Time + + store *store.Store + api ipc.CoreAPI // the intake-decorated adapter, same as the daemon builds + bus *event.Bus + handler *reactiveHandler + tick *tickLoop + sink *recordingSink + llm *scriptedLLM + + praxis *fakeServer + nexus *fakeServer + hexis *fakeServer + + // transcript — everything that happened, in order. Printed on failure so a + // broken scenario is diagnosable without a debugger. + transcript []string + replies []string +} + +// recordingSink captures every send, mutex-guarded (the tick loop dispatches +// from its own goroutine in production and the race detector is on here). +type recordingSink struct { + mu sync.Mutex + sends []delivery.Sendable +} + +func (s *recordingSink) Send(_ context.Context, d delivery.Sendable) error { + s.mu.Lock() + defer s.mu.Unlock() + s.sends = append(s.sends, d) + return nil +} + +func (s *recordingSink) all() []delivery.Sendable { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]delivery.Sendable, len(s.sends)) + copy(out, s.sends) + return out +} + +func (s *recordingSink) count() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.sends) +} + +// scriptedLLM stands in for llama-server on BOTH contracts the resident model +// serves: grammar-constrained routing and unconstrained phrasing. +// +// It is not a stub that ignores its input — a scenario that scripts an answer +// for "что я пропустил" and gets asked something else must fail, not silently +// return the wrong intent. An unmatched call returns an error, and the router +// then falls through to the classifier cascade exactly as it does in +// production when llama-server is unreachable. That fall-through is itself +// worth exercising: it is the failure floor CLAUDE.md refuses to let rot. +type scriptedLLM struct { + mu sync.Mutex + entries []scriptEntry + calls []llm.Req +} + +func (s *scriptedLLM) Complete(_ context.Context, r llm.Req) (string, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.calls = append(s.calls, r) + routing := r.Grammar != "" + for _, e := range s.entries { + if e.Match != "" && !strings.Contains(strings.ToLower(r.User), strings.ToLower(e.Match)) { + continue + } + if routing && e.Route != "" { + return e.Route, nil + } + if !routing && e.Reply != "" { + return e.Reply, nil + } + } + return "", fmt.Errorf("simulator: no scripted %s answer for %q", + map[bool]string{true: "route", false: "reply"}[routing], truncateRunes(r.User, 60)) +} + +// --------------------------------------------------------------------------- +// Building the world +// --------------------------------------------------------------------------- + +func newSimWorld(t *testing.T, sc scenario) *simWorld { + t.Helper() + + start, err := time.Parse(time.RFC3339, sc.Start) + if err != nil { + t.Fatalf("scenario %q: bad start %q: %v", sc.Name, sc.Start, err) + } + clock := newFakeClock(start) + + st := newTestStore(t) + bus := event.NewBus(512) + // The same decorator the daemon wires, on the same clock: intake in a + // replay is journalled exactly as it is in production. + api := newIntakeAPI(ipc.NewStoreAPI(st), bus, clock.Now) + + sink := &recordingSink{} + rules := loop.DefaultRules() + gatherer := loop.NewGatherer(st, rules) + dispatcher := delivery.NewDispatcher(delivery.Config{ + Voice: sink, Ntfy: sink, Telegram: sink, Nudges: st, Reminders: st, + }) + tl := newTickLoop(st, gatherer, dispatcher, phraser.NewStub(), rules, + time.Minute, 5*time.Minute, 0, nil, nil, nil, nil) + + scripted := &scriptedLLM{entries: sc.Script} + + w := &simWorld{ + t: t, clock: clock, loc: start.Location(), start: start, + store: st, api: api, bus: bus, tick: tl, sink: sink, llm: scripted, + } + + // Ecosystem fakes, wired only when the scenario supplies a body — a box + // with no praxis block has no praxis client, and a scenario must be able to + // reproduce that. + eco := &ecosystemWiring{} + if sc.Praxis != "" { + w.praxis = newFakePraxis(t, sc.Praxis) + eco.praxis = newPraxisClient(w.praxis.URL) + } + if sc.Nexus != "" { + w.nexus = newFakeNexus(t, sc.Nexus) + } + if sc.Hexis != "" { + w.hexis = newFakeHexis(t, sc.Hexis, fixtureHexisExecuted("exec_1", "completed")) + } + + // The router: the same cascade the daemon builds — stage-0 grammars, the + // LLM router on the scripted model, the classifier underneath. Keeping the + // classifier in is deliberate; it is the failure floor, and a scenario that + // scripts no route for an utterance exercises it. + emb := router.NewHashEmbedder(1024) + matcher := tool.NewMatcher(nil) + rtr := buildRouter(emb, matcher, config.DefaultRouterThreshold, router.NewLLMRouter(scripted)) + + w.handler = &reactiveHandler{ + stt: simTranscriber{}, + tts: simSynthesizer{}, + router: rtr, + embedder: emb, + api: api, + matcher: matcher, + phraser: phraser.NewStub(), + replier: newLLMReplier(scripted, nil), + now: clock.Now, + memStore: st.VectorMemory(), + dataStore: st, + queryMinScore: config.DefaultQueryMinScore, + queryMinMargin: config.DefaultQueryMinMargin, + timeParser: router.StubDateTimeParser{}, + dialogueSessions: dialogue.NewSessionStore(time.Hour), + clarifyStore: dialogue.NewClarifyStore(time.Hour), + clarifyMaxAttempts: dialogue.DefaultMaxAttempts, + ecosystem: eco, + } + return w +} + +// simTranscriber — the STT seam. Deterministic by construction: it returns the +// text the harness parked for this step, so the pipeline under test is +// "audio arrives → a turn runs", not "whisper heard correctly". Transcription +// accuracy is cmd/mavsttd/golden_test.go's job (#288 tier 1), and duplicating +// it here would make every scenario depend on a 500 MB model. +type simTranscriber struct{ text string } + +func (s simTranscriber) Transcribe(_ context.Context, _ audio.Audio) (string, float64, error) { + return s.text, 1.0, nil +} + +// simSynthesizer — the TTS seam. A scenario asserts on what Maven SAID, which +// is the reply text; the waveform is not the artefact under test. +type simSynthesizer struct{} + +func (simSynthesizer) Synthesize(_ context.Context, _ string) (audio.Audio, error) { + return audio.Audio{Format: audio.PCM16kMono}, nil +} + +// --------------------------------------------------------------------------- +// Running +// --------------------------------------------------------------------------- + +func (w *simWorld) logf(format string, args ...any) { + w.transcript = append(w.transcript, + fmt.Sprintf("%s %s", w.clock.Now().In(w.loc).Format("15:04:05"), fmt.Sprintf(format, args...))) +} + +// dump prints the whole transcript. Called on any failure — a scenario that +// broke on step 7 is unreadable without the six steps before it. +func (w *simWorld) dump() { + w.t.Logf("--- replay transcript ---\n%s", strings.Join(w.transcript, "\n")) +} + +// advanceTo moves the clock to the step's offset. Time only ever moves +// FORWARD: a scenario with steps out of order is a bug in the scenario, and +// silently reordering it would hide the bug. +func (w *simWorld) advanceTo(at string) { + w.t.Helper() + if at == "" { + return + } + target := w.timeOf(at) + now := w.clock.Now() + if target.Before(now) { + w.t.Fatalf("step at %s goes backwards from %s — scenario steps must be in order", + at, now.In(w.loc).Format("15:04:05")) + } + w.clock.Advance(target.Sub(now)) +} + +// timeOf resolves an "HH:MM" or "HH:MM:SS" step offset against the scenario's +// start day and location. +func (w *simWorld) timeOf(at string) time.Time { + w.t.Helper() + layout := "15:04" + if strings.Count(at, ":") == 2 { + layout = "15:04:05" + } + hm, err := time.Parse(layout, at) + if err != nil { + w.t.Fatalf("bad step time %q: %v", at, err) + } + return time.Date(w.start.Year(), w.start.Month(), w.start.Day(), + hm.Hour(), hm.Minute(), hm.Second(), 0, w.loc) +} + +func (w *simWorld) run(sc scenario) { + ctx := context.Background() + for i, s := range sc.Steps { + w.advanceTo(s.At) + if s.Note != "" { + w.logf("# %s", s.Note) + } + sendsBefore := w.sink.count() + callsBefore := w.callCount() + eventsBefore := w.bus.Len() + + w.stimulate(ctx, s) + w.assert(i, s, sendsBefore, callsBefore, eventsBefore) + } +} + +func (w *simWorld) stimulate(ctx context.Context, s step) { + if s.Fault != 0 || s.ClearFault { + for _, fs := range []*fakeServer{w.praxis, w.nexus, w.hexis} { + if fs != nil { + fs.SetFault(s.Fault) + } + } + w.logf("fault=%d on every ecosystem fake", s.Fault) + } + + switch { + case s.Say != "": + reply := w.handler.runTurn(ctx, s.Say) + w.replies = append(w.replies, reply) + w.logf("он: %s", s.Say) + w.logf("она: %s", reply) + + case s.Audio != "": + text := w.audioText(s.Audio) + // Swap in a transcriber parked with this step's text, then run the same + // push-to-talk entry point the voice client calls. + w.handler.stt = simTranscriber{text: text} + resp, err := w.handler.HandlePushToTalk(ctx, voicePTT(), 0) + if err != nil { + w.t.Fatalf("push-to-talk on %s: %v", s.Audio, err) + } + w.replies = append(w.replies, resp.ReplyText) + w.logf("[wav %s → %q]", filepath.Base(s.Audio), text) + w.logf("она: %s", resp.ReplyText) + + case s.Signal != nil: + w.write(ctx, *s.Signal, w.clock.Now()) + w.logf("сигнал: %s=%s (%s)", s.Signal.Key, s.Signal.Value, s.Signal.Source) + + case s.Arrive != nil: + w.arrive(ctx, *s.Arrive) + + case s.Tick: + w.tick.tick(ctx, w.clock.Now()) + w.logf("tick") + } +} + +func (w *simWorld) write(ctx context.Context, sig signalStep, ts time.Time) { + w.t.Helper() + kind := sig.Kind + if kind == "" { + kind = "env" + } + if _, err := w.api.WriteFact(ctx, ipc.WriteFactReq{ + Ts: ts, Kind: kind, Key: sig.Key, Value: sig.Value, Source: sig.Source, Confidence: 1.0, + }); err != nil { + w.t.Fatalf("write fact %s: %v", sig.Key, err) + } +} + +func (w *simWorld) arrive(ctx context.Context, a arriveStep) { + w.t.Helper() + // AsOf is when the thing HAPPENED, which for a feed item or a relayed + // notification is usually earlier than when Maven heard about it. It does + // not move the clock — only the timestamp on the row and the envelope. + ts := w.clock.Now() + if a.AsOf != "" { + ts = w.timeOf(a.AsOf) + } + switch { + case a.Fact != nil: + f := *a.Fact + if f.Source == "" { + f.Source = a.Source + } + w.write(ctx, f, ts) + w.logf("пришло: факт %s=%s (%s)", f.Key, f.Value, f.Source) + case a.Note != nil: + if _, err := w.api.WriteNote(ctx, ts, a.Note.Text, nil, a.Source); err != nil { + w.t.Fatalf("write note from %s: %v", a.Source, err) + } + w.logf("пришло: заметка от %s — %s", a.Source, truncateRunes(a.Note.Text, 60)) + case a.Task != nil: + status := a.Task.Status + if status == "" { + status = store.TaskCandidate + } + if _, err := w.api.CaptureTask(ctx, ipc.CaptureTaskReq{ + Text: a.Task.Text, Source: a.Source, Evidence: a.Task.Evidence, Status: status, Ts: ts, + }); err != nil { + w.t.Fatalf("capture task from %s: %v", a.Source, err) + } + w.logf("пришло: задача от %s — %s", a.Source, a.Task.Text) + default: + w.t.Fatalf("arrive step from %s carries nothing", a.Source) + } +} + +// audioText resolves a scenario's WAV reference to the text the fixture is +// known to contain, by reading cmd/mavsttd's golden manifest (#288's format, +// reused rather than duplicated). An unknown reference fails the scenario +// rather than quietly transcribing to "". +func (w *simWorld) audioText(ref string) string { + w.t.Helper() + manifest := filepath.Join("..", "mavsttd", "testdata", "golden_v1.json") + raw, err := os.ReadFile(manifest) + if err != nil { + w.t.Fatalf("audio step %q: reading %s: %v", ref, manifest, err) + } + var m struct { + Cases []struct { + Name string `json:"name"` + WAV string `json:"wav"` + Text string `json:"text"` + } `json:"cases"` + } + if err := json.Unmarshal(raw, &m); err != nil { + w.t.Fatalf("audio step %q: parsing %s: %v", ref, manifest, err) + } + for _, c := range m.Cases { + if c.Name == ref || c.WAV == ref { + return c.Text + } + } + w.t.Fatalf("audio step %q: no such case in %s", ref, manifest) + return "" +} + +func voicePTT() voice.PushToTalkReq { + return voice.PushToTalkReq{Audio: audio.Audio{Format: audio.PCM16kMono}} +} + +// callCount — how many requests every wired ecosystem fake has seen. +func (w *simWorld) callCount() int { + n := 0 + for _, fs := range []*fakeServer{w.praxis, w.nexus, w.hexis} { + if fs != nil { + n += len(fs.Requests()) + } + } + return n +} + +func (w *simWorld) callPaths() []string { + var out []string + for _, fs := range []*fakeServer{w.praxis, w.nexus, w.hexis} { + if fs == nil { + continue + } + for _, r := range fs.Requests() { + out = append(out, r.Method+" "+r.Path) + } + } + return out +} + +// --------------------------------------------------------------------------- +// Assertions +// --------------------------------------------------------------------------- + +func (w *simWorld) assert(i int, s step, sendsBefore, callsBefore, eventsBefore int) { + w.t.Helper() + where := fmt.Sprintf("step %d (%s)", i+1, s.At) + if s.Note != "" { + where += " " + s.Note + } + fail := func(format string, args ...any) { + w.dump() + w.t.Errorf("%s: %s", where, fmt.Sprintf(format, args...)) + } + + lastReply := "" + if len(w.replies) > 0 { + lastReply = w.replies[len(w.replies)-1] + } + for _, want := range s.ExpectReply { + if !containsFold(lastReply, want) { + fail("reply %q does not contain %q", lastReply, want) + } + } + for _, unwanted := range s.ExpectNotReply { + if containsFold(lastReply, unwanted) { + fail("reply %q contains %q and must not", lastReply, unwanted) + } + } + + sent := w.sink.all() + for _, want := range s.ExpectSent { + if !anyContains(sendableTexts(sent), want) { + fail("nothing sent mentions %q; sent so far: %v", want, sendableTexts(sent)) + } + } + // Scoped to this step on purpose: "nothing was sent BECAUSE OF THIS" is the + // question a not-a-nag constraint asks. + if s.ExpectNoSend && len(sent) > sendsBefore { + fail("expected nothing to be sent, got %v", sendableTexts(sent[sendsBefore:])) + } + + paths := w.callPaths() + for _, want := range s.ExpectCalled { + if !anyContains(paths, want) { + fail("no ecosystem call matches %q; calls so far: %v", want, paths) + } + } + for _, unwanted := range s.ExpectNotCalled { + if anyContains(paths[callsBefore:], unwanted) { + fail("an ecosystem call matched %q and must not have: %v", unwanted, paths[callsBefore:]) + } + } + + evs := w.bus.Recent(0) + for _, want := range s.ExpectEvents { + if !anyContains(eventLines(evs), want) { + fail("no intake event matches %q; journal: %v", want, eventLines(evs)) + } + } + if s.ExpectNoEvents && w.bus.Len() > eventsBefore { + fail("expected nothing to arrive, journal grew to %d", w.bus.Len()) + } +} + +func sendableTexts(sends []delivery.Sendable) []string { + out := make([]string, 0, len(sends)) + for _, s := range sends { + out = append(out, fmt.Sprintf("[%s] %s", s.RuleName, s.Body)) + } + return out +} + +func eventLines(evs []event.Event) []string { + out := make([]string, 0, len(evs)) + for _, e := range evs { + out = append(out, fmt.Sprintf("%s/%s %s %s", e.Source, e.Kind, e.Title, e.Body)) + } + return out +} + +func containsFold(hay, needle string) bool { + return strings.Contains(strings.ToLower(hay), strings.ToLower(needle)) +} + +func anyContains(hay []string, needle string) bool { + for _, h := range hay { + if containsFold(h, needle) { + return true + } + } + return false +} + +// --------------------------------------------------------------------------- +// The test +// --------------------------------------------------------------------------- + +const scenarioDir = "testdata/scenarios" + +// TestSimulatorScenarios replays every scenario file. Adding a scenario is +// adding a JSON file — no Go change, which is the property that makes this +// cheap enough to actually use. +func TestSimulatorScenarios(t *testing.T) { + entries, err := os.ReadDir(scenarioDir) + if err != nil { + t.Fatalf("reading %s: %v", scenarioDir, err) + } + var ran int + for _, ent := range entries { + if ent.IsDir() || !strings.HasSuffix(ent.Name(), ".json") { + continue + } + ran++ + name := strings.TrimSuffix(ent.Name(), ".json") + t.Run(name, func(t *testing.T) { + sc := loadScenario(t, filepath.Join(scenarioDir, ent.Name())) + w := newSimWorld(t, sc) + w.run(sc) + if testing.Verbose() { + w.dump() + } + }) + } + if ran == 0 { + t.Fatalf("no scenarios in %s — the harness would pass vacuously", scenarioDir) + } +} + +func loadScenario(t *testing.T, path string) scenario { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("reading %s: %v", path, err) + } + var sc scenario + dec := json.NewDecoder(strings.NewReader(string(raw))) + dec.DisallowUnknownFields() // a typo'd assertion key must fail, not be ignored + if err := dec.Decode(&sc); err != nil { + t.Fatalf("parsing %s: %v", path, err) + } + if sc.SchemaVersion != 1 { + t.Fatalf("%s: schema_version = %d, want 1", path, sc.SchemaVersion) + } + if sc.Name == "" || sc.Start == "" || len(sc.Steps) == 0 { + t.Fatalf("%s: a scenario needs a name, a start and at least one step", path) + } + return sc +} + +// TestSimulatorIsDeterministic replays one scenario twice and requires an +// identical transcript. This is the property the whole task rests on: if a +// time.Now() creeps into a replayed path, two runs diverge and this fails. +func TestSimulatorIsDeterministic(t *testing.T) { + path := filepath.Join(scenarioDir, "morning_missed.json") + sc := loadScenario(t, path) + + transcriptOf := func() string { + w := newSimWorld(t, sc) + w.run(sc) + return strings.Join(w.transcript, "\n") + } + first := transcriptOf() + second := transcriptOf() + if first != second { + t.Errorf("two replays of the same scenario diverged:\n--- first ---\n%s\n--- second ---\n%s", first, second) + } + // And the transcript's own timestamps must be the scenario's, not today's. + if strings.Contains(first, time.Now().Format("15:04")) && !strings.Contains(sc.Start, time.Now().Format("15:04")) { + t.Error("transcript carries the wall clock — something in the replay path read time.Now()") + } +} + +// TestSimulatorRefusesBackwardsSteps guards the one scenario-authoring mistake +// that would silently produce a meaningless run. +func TestSimulatorRefusesBackwardsSteps(t *testing.T) { + // Not table-driven through run() because advanceTo calls t.Fatalf; this + // checks the ordering arithmetic directly. + sc := scenario{SchemaVersion: 1, Name: "x", Start: "2026-08-01T08:30:00+03:00", + Steps: []step{{At: "09:00"}}} + w := newSimWorld(t, sc) + w.advanceTo("09:00") + if got := w.clock.Now().In(w.loc).Format("15:04"); got != "09:00" { + t.Fatalf("clock at %s after advancing to 09:00", got) + } + w.advanceTo("09:30") + if got := w.clock.Now().In(w.loc).Format("15:04"); got != "09:30" { + t.Fatalf("clock at %s after advancing to 09:30", got) + } +} diff --git a/cmd/mavend/testdata/scenarios/evening_degraded.json b/cmd/mavend/testdata/scenarios/evening_degraded.json new file mode 100644 index 0000000..92bbbb0 --- /dev/null +++ b/cmd/mavend/testdata/scenarios/evening_degraded.json @@ -0,0 +1,67 @@ +{ + "schema_version": 1, + "name": "evening_degraded", + "description": "The tier-2 pipeline case #288 deferred here, plus degraded mode. A golden WAV goes in at the microphone end and comes out as a written fact, and then the ecosystem starts answering 503 and the proactive loop has to stay quiet instead of falling over. The audio step asserts the PIPELINE — mic to STT seam to router to store to TTS — not whisper's accuracy; cmd/mavsttd/golden_test.go owns accuracy.", + "start": "2026-08-01T21:00:00+03:00", + "praxis_attention": "[{\"id\":\"item_1\",\"title\":\"medicine not taken\",\"importance\":3.0,\"rule\":\"evening_medicine\"}]", + "script": [ + { + "match": "выпил воды", + "route": "[{\"intent\":\"fact\",\"key\":\"water\",\"value\":\"выпил\"}]" + }, + { + "match": "записала факт: water", + "reply": "{\"response\":\"Записала, что ты выпил воды.\",\"mood\":\"neutral\"}" + }, + { + "match": "", + "route": "[{\"intent\":\"chat\",\"text\":\"привет\"}]", + "reply": "{\"response\":\"Я рада тебя слышать.\",\"mood\":\"happy\"}" + } + ], + "steps": [ + { + "at": "21:00", + "note": "he speaks. The whole voice path runs: push-to-talk, the STT seam parked with the golden transcript, the real router, the real store write, the phrasing contract.", + "audio": "ru_fact", + "expect_reply_contains": ["записала"], + "expect_reply_lacks": ["записал,", "милый", "ваш"], + "expect_events": ["water"] + }, + { + "at": "21:05", + "note": "a healthy tick with him just having spoken stays silent", + "tick": true, + "expect_no_send": true + }, + { + "at": "21:10", + "note": "the ecosystem goes down", + "fault": 503 + }, + { + "at": "21:15", + "note": "a tick against a dead ecosystem must degrade, not send half a thought", + "tick": true, + "expect_no_send": true, + "expect_no_events": true + }, + { + "at": "21:20", + "note": "intake keeps working while the ecosystem is down — a write does not depend on it", + "arrive": { + "source": "rss:tech", + "note": { "text": "Патч 6.19.1 [tech]\nисправления\nhttps://example.org/b" } + }, + "expect_events": ["rss:tech"], + "expect_no_send": true + }, + { + "at": "21:25", + "note": "recovery", + "clear_fault": true, + "tick": true, + "expect_no_send": true + } + ] +} diff --git a/cmd/mavend/testdata/scenarios/morning_missed.json b/cmd/mavend/testdata/scenarios/morning_missed.json new file mode 100644 index 0000000..af01fad --- /dev/null +++ b/cmd/mavend/testdata/scenarios/morning_missed.json @@ -0,0 +1,96 @@ +{ + "schema_version": 1, + "name": "morning_missed", + "description": "The scenario from Vikunja #284's description, replayed. He appears at 08:30, things arrive through the morning while he is at the desk, and at 08:50 he asks what he missed. The assertions are as much about what did NOT happen — nothing was sent at him unprompted — as about what she said.", + "start": "2026-08-01T08:30:00+03:00", + "praxis_attention": "[{\"id\":\"item_1\",\"title\":\"medicine not taken\",\"importance\":3.0,\"rule\":\"morning_medicine\"}]", + "script": [ + { + "match": "выпил воды", + "route": "[{\"intent\":\"fact\",\"key\":\"water\",\"value\":\"выпил\"}]" + }, + { + "match": "записала факт: water", + "reply": "{\"response\":\"Записала, что ты выпил воды.\",\"mood\":\"neutral\"}" + }, + { + "match": "что я пропустил", + "route": "[{\"intent\":\"query\",\"text\":\"что я пропустил\"}]" + }, + { + "match": "", + "route": "[{\"intent\":\"chat\",\"text\":\"привет\"}]", + "reply": "{\"response\":\"Я рада тебя слышать.\",\"mood\":\"happy\"}" + } + ], + "steps": [ + { + "at": "08:30", + "note": "he appears at the desk", + "signal": { "key": "desk_active", "value": "true", "source": "infer:hyprland" }, + "expect_events": ["infer:hyprland"], + "expect_no_send": true + }, + { + "at": "08:32", + "note": "a feed item arrives, published half an hour ago", + "arrive": { + "source": "rss:tech", + "as_of": "08:02", + "note": { "text": "Вышло ядро 6.19 [tech]\nкраткое содержание\nhttps://example.org/a" } + }, + "expect_events": ["rss:tech"], + "expect_no_send": true + }, + { + "at": "08:35", + "note": "the mail reader extracts a candidate — a candidate is never spoken", + "arrive": { + "source": "email:inbox", + "task": { "text": "продлить домен", "evidence": "Домен истекает через 7 дней" } + }, + "expect_events": ["email:inbox", "продлить домен"], + "expect_no_send": true + }, + { + "at": "08:40", + "note": "the work calendar signal — a relayed notification, below full confidence", + "arrive": { + "source": "ambient:notif", + "fact": { + "key": "calendar_event_20260801_планёрка", + "value": "10:00-11:00 планёрка" + } + }, + "expect_events": ["ambient:notif", "планёрка"], + "expect_no_send": true + }, + { + "at": "08:45", + "note": "a tick with him present and nothing wrong must stay silent", + "tick": true, + "expect_no_send": true + }, + { + "at": "08:50", + "note": "he asks. The query path answers from local recall only: nothing stored clears the score gate, so she refuses rather than inventing a morning summary, and the replier is never reached. That refusal is the no-hallucination floor and this step pins it.", + "say": "что я пропустил?", + "expect_reply_contains": ["не знаю"], + "expect_reply_lacks": ["рад ", "милый", "ваш"] + }, + { + "at": "08:55", + "note": "stating a fact writes it and says so, in the feminine", + "say": "я выпил воды", + "expect_reply_contains": ["записала"], + "expect_reply_lacks": ["записал,", "милый"], + "expect_events": ["water"] + }, + { + "at": "09:00", + "note": "a second tick, still nothing unprompted", + "tick": true, + "expect_no_send": true + } + ] +}