Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d773f1f72b | |||
| 80ac7579fb | |||
| bb6cb6d185 | |||
| a9067a5754 | |||
| 787cc56522 | |||
| 86dcd99de2 | |||
| 554181ccbd | |||
| 58635f1a69 | |||
| fd3d063e02 |
@@ -77,8 +77,8 @@ Pure-Go packages (`router`, `memory`, `mavweb`, …) run under a plain `go test
|
|||||||
| `mavweb` | HTTP UI + PWA (`/dash`, `/history`, `/trace`, `/notifications`, `/tools`); WebAuthn auth. Connects to mavend's socket. |
|
| `mavweb` | HTTP UI + PWA (`/dash`, `/history`, `/trace`, `/notifications`, `/tools`); WebAuthn auth. Connects to mavend's socket. |
|
||||||
| `mavsttd` | Speech-to-text (whisper.cpp, CGO). |
|
| `mavsttd` | Speech-to-text (whisper.cpp, CGO). |
|
||||||
| `mavttsd` | Text-to-speech (piper subprocess). |
|
| `mavttsd` | Text-to-speech (piper subprocess). |
|
||||||
| `mavwaked` | Wake-word / VAD gate. |
|
| `mavwaked` | Wake-word / VAD gate. **Not on homesrv** — see below. |
|
||||||
| `mavenclient` | Voice loop client (mic → stt → core → tts). |
|
| `mavenclient` | Voice loop client (mic → stt → core → tts). **Not on homesrv** — see below. |
|
||||||
| `mavpoll` | Telegram long-poll reach. |
|
| `mavpoll` | Telegram long-poll reach. |
|
||||||
| `mavcaldav` | CalDAV calendar sync. |
|
| `mavcaldav` | CalDAV calendar sync. |
|
||||||
| `mavmaild` | Mail reader (IMAP, read-only). Holds the IMAP password; core never sees it. |
|
| `mavmaild` | Mail reader (IMAP, read-only). Holds the IMAP password; core never sees it. |
|
||||||
@@ -87,6 +87,15 @@ Daemons are wired socket-to-socket, not linked. `internal/ipc` is the client/ser
|
|||||||
protocol; the config in `deploy/mavend.json` (with `${VAR}` env expansion from gitignored
|
protocol; the config in `deploy/mavend.json` (with `${VAR}` env expansion from gitignored
|
||||||
`deploy/telegram.env`) sets socket paths, model paths, and the phraser/embedder blocks.
|
`deploy/telegram.env`) sets socket paths, model paths, and the phraser/embedder blocks.
|
||||||
|
|
||||||
|
**Seven of the nine run on homesrv. `mavwaked` and `mavenclient` do not, and that is the
|
||||||
|
decision, not an oversight** (Vikunja #463, `docs/plans/17-where-the-voice-loop-runs.md`).
|
||||||
|
homesrv has a microphone — it is a laptop — but it is in the wrong room, so a wake-word
|
||||||
|
daemon there listens to nobody. They belong on a client machine where he is standing.
|
||||||
|
`ipc.Dial` already takes `tcp://host:port?token=...` through the netaddr seam, so nothing
|
||||||
|
needs building to allow it, but no such machine exists yet. **The consequence: the wake
|
||||||
|
word and the VAD gate are covered by unit tests and by nothing else, and no amount of
|
||||||
|
sitting at the box changes that.** Push-to-talk through `/dash` is what QA actually covers.
|
||||||
|
|
||||||
## The ecosystem: Nexus, Praxis, Hexis
|
## The ecosystem: Nexus, Praxis, Hexis
|
||||||
|
|
||||||
Maven is one of four services. It owns conversation and personal memory. It does not
|
Maven is one of four services. It owns conversation and personal memory. It does not
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ PIPER_BIN := $(shell pwd)/deps/piper/piper
|
|||||||
PIPER_MODEL := $(shell pwd)/models/tts/ru_RU-irina-medium.onnx
|
PIPER_MODEL := $(shell pwd)/models/tts/ru_RU-irina-medium.onnx
|
||||||
PIPER_ESPEAK := $(shell pwd)/deps/piper/espeak-ng-data
|
PIPER_ESPEAK := $(shell pwd)/deps/piper/espeak-ng-data
|
||||||
|
|
||||||
.PHONY: simulate stt-fixtures test-stt-golden all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test fmt-check vet run-stt run-tts run-web download-embedder deps-go eval-router eval-recall eval-phrasing eval-models build-gpud
|
.PHONY: simulate stt-fixtures test-stt-golden all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test fmt-check vet run-stt run-tts run-web download-embedder deps-go eval-router eval-reach eval-recall eval-phrasing eval-models build-gpud
|
||||||
|
|
||||||
all: build
|
all: build
|
||||||
|
|
||||||
@@ -119,6 +119,14 @@ MAVEN_ONNX_LIB ?= $(shell pwd)/deps/onnxruntime-linux-x64-1.26.0/lib/libonnxrunt
|
|||||||
eval-router:
|
eval-router:
|
||||||
MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 ./internal/router/eval/
|
MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 ./internal/router/eval/
|
||||||
|
|
||||||
|
# eval-reach — score the held-out ecosystem reach fixture (internal/router/eval,
|
||||||
|
# ru_ecosystem_v1.json). Answers "does a real Russian utterance actually arrive
|
||||||
|
# at Praxis or Hexis", which routing accuracy alone does not say. Same
|
||||||
|
# MAVEN_ONNX_LIB deal as eval-router; without it only the deterministic hash
|
||||||
|
# ratchet runs. Vikunja #405.
|
||||||
|
eval-reach:
|
||||||
|
MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 -run 'Reach|Praxis' ./internal/router/eval/
|
||||||
|
|
||||||
# eval-recall — score the held-out note-recall fixture (internal/memory/recalleval).
|
# eval-recall — score the held-out note-recall fixture (internal/memory/recalleval).
|
||||||
# Answers "can she find the note again when it matters": recall@1, recall@3,
|
# Answers "can she find the note again when it matters": recall@1, recall@3,
|
||||||
# false recall and the query_min_score sweep. Same MAVEN_ONNX_LIB deal as
|
# false recall and the query_min_score sweep. Same MAVEN_ONNX_LIB deal as
|
||||||
|
|||||||
@@ -0,0 +1,100 @@
|
|||||||
|
# Ecosystem reach, measured — 2026-08-04
|
||||||
|
|
||||||
|
Vikunja #405. Measured at `86dcd99` on the 30-case held-out fixture
|
||||||
|
`internal/router/eval/ru_ecosystem_v1.json`, scored by `make eval-reach`.
|
||||||
|
|
||||||
|
**Praxis reach is zero. Not low — zero, on all twelve cases, under both embedders.**
|
||||||
|
|
||||||
|
## What was measured
|
||||||
|
|
||||||
|
Reach is where an utterance *arrives*, not what it achieves. The derivation is in
|
||||||
|
`internal/router/eval/reach.go` and mirrors `actionAct` in `cmd/mavend/actions_act.go`:
|
||||||
|
|
||||||
|
- **Praxis** needs `IntentAct` and a fn slot whose value is one of the aliases in
|
||||||
|
`praxisCapabilities`.
|
||||||
|
- **Hexis** needs `IntentAct` and non-empty `Slots.Text`. It also fires from
|
||||||
|
`hexisBeforeClarify`, so a clarified act with text reaches Hexis before the clarify
|
||||||
|
question is ever asked.
|
||||||
|
- Everything else stays inside Maven.
|
||||||
|
|
||||||
|
The fixture holds 10 Hexis cases, 12 Praxis cases and 8 negatives. The negatives carry
|
||||||
|
the expensive direction: an utterance that reaches a mutating path it had no business
|
||||||
|
reaching is worse than one that never arrives, because he never gets asked about it.
|
||||||
|
|
||||||
|
## The numbers
|
||||||
|
|
||||||
|
| Path | Reached the right place | Missed | Overreach | Wrong service | p50 |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| classifier + hash embedder | 7/30 (23.3%) | 22 | 1 | 0 | 14.8µs |
|
||||||
|
| classifier + e5-small (deployed) | 16/30 (53.3%) | 11 | 1 | 2 | 23.8ms |
|
||||||
|
|
||||||
|
Split by target, on the deployed embedder:
|
||||||
|
|
||||||
|
| Want | Passed |
|
||||||
|
|---|---|
|
||||||
|
| hexis | 9/10 |
|
||||||
|
| none | 7/8 |
|
||||||
|
| **praxis** | **0/12** |
|
||||||
|
|
||||||
|
## The finding
|
||||||
|
|
||||||
|
Hexis reach is fine. Nine of ten act-shaped home and homelab utterances land on the
|
||||||
|
entity resolver, which is what the gate was built to do: it only needs the intent and
|
||||||
|
some text, and both the act grammar and the embedder produce those.
|
||||||
|
|
||||||
|
Praxis reach is structurally impossible from free Russian, and the fixture makes that
|
||||||
|
visible for the first time. `handlePraxisAct` dispatches on exact equality between
|
||||||
|
`Slots.Fn` and a capability alias. The fn slot is filled by `DefaultActMatcher`, whose
|
||||||
|
allowlist is the deployment's enabled tool names — `перезапусти`, `выключи`, and so on.
|
||||||
|
No Praxis alias is in that list, so no utterance can ever put one in the slot. The
|
||||||
|
Russian aliases in `praxisCapabilities` (`готово`, `принято`, `игнорировать`) read as if
|
||||||
|
they match speech and they do not: they are compared against a fn slot, never against
|
||||||
|
the utterance.
|
||||||
|
|
||||||
|
That means the whole lifecycle half of the Praxis contract — acknowledge, resolve,
|
||||||
|
ignore, pin — has no voice path at all. Attention and changes have none either.
|
||||||
|
|
||||||
|
Three of the twelve got as far as the wrong place, which is the same defect seen from the
|
||||||
|
other side: `"готово, закрывай"` and `"как дела у праксиса"` route to act with text, so
|
||||||
|
they fall past the Praxis check into the Hexis one and go to entity resolution instead.
|
||||||
|
|
||||||
|
## The live services
|
||||||
|
|
||||||
|
All three are up and answer. With `no_proxy` set for the loopback (the host's `http_proxy`
|
||||||
|
answers 503 for 127.0.0.1, which is the same trap `llmrouter_test.go` documents):
|
||||||
|
|
||||||
|
```
|
||||||
|
127.0.0.1:8989/health -> {"status":"ok"} praxis
|
||||||
|
127.0.0.1:9740/health -> {"status":"ok"} nexus
|
||||||
|
127.0.0.1:9741/health -> {"status":"ok"} hexis
|
||||||
|
127.0.0.1:8989/api/v1/tools/attention?limit=3 -> []
|
||||||
|
127.0.0.1:8989/api/v1/tools/changes?limit=3 -> []
|
||||||
|
```
|
||||||
|
|
||||||
|
So the boundary is not the problem, and an end-to-end run today would add nothing: both
|
||||||
|
Praxis feeds are empty, so even a perfect reach score would produce "ничего не требует
|
||||||
|
внимания". The gap is entirely on Maven's side of the wire.
|
||||||
|
|
||||||
|
## Not measured
|
||||||
|
|
||||||
|
**The resident model.** The LLM router is the deployed default, and these numbers are the
|
||||||
|
classifier only. `mavend` spawns its llama-server on a container-local port
|
||||||
|
(127.0.0.1:40063 inside `maven-mavend-1`), unreachable from the host, and the workstation
|
||||||
|
at 192.168.1.105:8080 was down. The classifier is the failure floor and it is what always
|
||||||
|
answers, so the floor is worth knowing on its own — but the LLM router could fill the fn
|
||||||
|
slot with a literal `list_attention`, since the grammar lets it emit any string. Whether
|
||||||
|
it does is the open question, and the fixture is ready for it.
|
||||||
|
|
||||||
|
## What this argues for
|
||||||
|
|
||||||
|
Not a new intent. The seven are frozen by prompt parity with the training workspace.
|
||||||
|
|
||||||
|
The cheap fix is stage 0: a grammar per Praxis capability that sets `Slots.Fn` to the
|
||||||
|
canonical arm name, the same trick `AgendaQueryGrammars` used to take agenda questions off
|
||||||
|
the model. It costs one regex per capability on every turn and it is deterministic, which
|
||||||
|
for a lifecycle verb is the right trade — "отметь это как сделанное" should never be a
|
||||||
|
similarity guess.
|
||||||
|
|
||||||
|
The second fix is smaller and separate: `entity_attention` aliases to grammar names only,
|
||||||
|
so scoped attention ("что там с нексусом") needs a grammar before it can be reached at
|
||||||
|
all.
|
||||||
@@ -103,9 +103,14 @@ query path.
|
|||||||
```
|
```
|
||||||
|
|
||||||
`Recognizes()` requires both `enabled` and a `model_path`, so a half-filled block reads as off
|
`Recognizes()` requires both `enabled` and a `model_path`, so a half-filled block reads as off
|
||||||
rather than as a capability that fails every turn. With `enabled` and no model the daemon still
|
rather than as a capability that fails every turn.
|
||||||
attaches the three methods — profiles can be created, listed and deleted — and logs that
|
|
||||||
recognition is blocked.
|
That gate was written, documented, and then never called. It is called now, and the behaviour it
|
||||||
|
describes changed with it. `enabled` with no `model_path` used to attach all three methods and log
|
||||||
|
that enrolment was on. Today `newSpeakerWiring` returns nil, so the methods are absent, and the log
|
||||||
|
says why: there is nothing to embed with, so enrol, list and forget would all be no-ops. That is
|
||||||
|
the one config shape where the operator most needs to be told otherwise, and it was the shape that
|
||||||
|
lied.
|
||||||
|
|
||||||
## Still open
|
## Still open
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,119 @@
|
|||||||
|
# Plan: The work board surface
|
||||||
|
|
||||||
|
**The decision Vikunja #431 asked for. Written 04-08-2026.**
|
||||||
|
|
||||||
|
**Verdict: build it, in a smaller shape than the task imagined.** The board is worth
|
||||||
|
moving out of the file. The intake form belongs on the `/tasks` page, not on the voice
|
||||||
|
path. The argument is not built, now or later.
|
||||||
|
|
||||||
|
## Why it is worth building
|
||||||
|
|
||||||
|
The reason is the one the task gives and it holds: company rules forbid pointing Claude
|
||||||
|
at work repos, and Maven is the one assistant on the box that work material may reach.
|
||||||
|
No telemetry, no cloud model, no third-party account. That is not a preference here, it
|
||||||
|
is the whole permission.
|
||||||
|
|
||||||
|
The build is also small, because most of it landed already:
|
||||||
|
|
||||||
|
| Piece | Where | State |
|
||||||
|
|---|---|---|
|
||||||
|
| task rows, dedupe, status lifecycle | `internal/store/migrations.go:149` and migration #15 | done |
|
||||||
|
| capture from speech, urgency stripped | `router.ParseTaskCapture`, `router.TaskCaptureGrammar` | done |
|
||||||
|
| recite the list on request | `router.IsTaskListQuery` | done |
|
||||||
|
| a page to read and change the board | `/tasks` in `cmd/mavweb` | done |
|
||||||
|
| counting a shape without judging it | `internal/memory/behavior.go` | done, as precedent |
|
||||||
|
| a proposal he reads when he chooses | `/routines`, the proposed-routine queue | done, as precedent |
|
||||||
|
|
||||||
|
`tasks` already carries `status` (candidate, open, done, dropped), `due_ts`, `weight`,
|
||||||
|
`source`, `evidence`, `ext_id` and `resolved_by`. Three things are missing. It has no
|
||||||
|
definition of done and no blocked-on. There is no way to edit a task after capture:
|
||||||
|
`SetTaskStatus` moves the status and nothing writes text, date or weight again. And
|
||||||
|
there is no grouping he controls, because order is computed by `tasks.Rank` alone.
|
||||||
|
|
||||||
|
## Where the form lives, and why not voice
|
||||||
|
|
||||||
|
The task asks the form to refuse a capture with no definition of done. That refusal
|
||||||
|
cannot live on the voice path, for two reasons.
|
||||||
|
|
||||||
|
**The parked-state mechanism is binary.** `resolveConfirm` in `cmd/mavend/confirm.go`
|
||||||
|
answers yes or no against a slot with a 90-second life. Filling four fields over four
|
||||||
|
turns is slot filling, which is a different mechanism and a new one. Nothing in the
|
||||||
|
daemon does it today.
|
||||||
|
|
||||||
|
**The definition of done is the worst possible field to dictate.** It is the one string
|
||||||
|
that has to be exact, because its whole purpose is to be unarguable later. Whisper
|
||||||
|
transcribing a sentence of Russian work vocabulary is where exactness goes to die, and
|
||||||
|
the capture path already had to strip a question mark that whisper invented.
|
||||||
|
|
||||||
|
So: voice captures a line and recites the list. The page is where a line becomes an
|
||||||
|
item with a definition of done, a blocked-on and a date. A captured line lands as
|
||||||
|
`candidate` and stays there until it is filled in, which is what `candidate` was for.
|
||||||
|
|
||||||
|
The refusal the task wants survives, moved: the page will not promote a candidate to
|
||||||
|
`open` without a definition of done, the same way `ParseTaskCapture` will not file a
|
||||||
|
marker with nothing after it. And the field must close on either outcome, so "it already
|
||||||
|
works" counts as complete. A definition of done that only one result satisfies is a wish.
|
||||||
|
|
||||||
|
## Does the stage-0 trick stretch
|
||||||
|
|
||||||
|
The task asks this before any shape is committed to. It was checked. The answer is
|
||||||
|
partly.
|
||||||
|
|
||||||
|
`TaskCaptureGrammar` matches every utterance and lets `ParseTaskCapture` decide inside
|
||||||
|
`Build`, keeping the intent at `note` and leaving the frozen seven-intent contract alone.
|
||||||
|
That trick stretches to **recite** and to **status change**: both are a marker plus a
|
||||||
|
referent, both are a lookup, and a status change is a small closed verb set over a list
|
||||||
|
he can see. It does not stretch to **intake**, because intake is not one utterance, and
|
||||||
|
it does not need to, because intake moved to the page.
|
||||||
|
|
||||||
|
One cost to name. Each such grammar matches everything and runs its parser on every
|
||||||
|
turn, ahead of the resident model. Two more of them is fine. A dozen would make stage 0
|
||||||
|
a second router with no evaluation behind it, and at that point the frozen enum is the
|
||||||
|
smaller problem.
|
||||||
|
|
||||||
|
## What is not built: the argument
|
||||||
|
|
||||||
|
Not now and not later behind a flag. The task is right about why, and
|
||||||
|
`internal/memory/behavior.go` already argued it for habits: a 1.7B asked whether evidence
|
||||||
|
proves anything will agree fluently and launder a guess into a decision. A wrong claim
|
||||||
|
about his work, stated confidently, is the most expensive kind of wrong Maven can be.
|
||||||
|
|
||||||
|
The line is the same line behaviour memory drew. She may **count**:
|
||||||
|
|
||||||
|
- no state change in eleven days
|
||||||
|
- blocked on a person, with no date
|
||||||
|
- four of nine waiting on two people
|
||||||
|
|
||||||
|
Those are queries over rows. She may not assess whether a build proves anything, whether
|
||||||
|
a blocker is real, or whether a task should be dropped.
|
||||||
|
|
||||||
|
## Persona
|
||||||
|
|
||||||
|
A progress tracker is a nag by default, and "not a nag" is hard. The line is already
|
||||||
|
drawn twice in the codebase and it is drawn the same way here:
|
||||||
|
|
||||||
|
- A date he set becomes a reminder. He set it, so it is not her raising it.
|
||||||
|
- A stall becomes a proposal he reads when he chooses, on a page, like `/routines`.
|
||||||
|
- Ask what is on the board and she recites. She never opens with it.
|
||||||
|
|
||||||
|
The day plan is the place to watch. `tickLoop.dayPlan` reads calendar events, pending
|
||||||
|
reminders and checklist facts, and it does not read tasks. Adding the board to the
|
||||||
|
morning nudge is exactly the move that turns this into a nag, so the board goes on the
|
||||||
|
page and into the answer when asked, and not into the unprompted morning message.
|
||||||
|
|
||||||
|
## The build, as tasks
|
||||||
|
|
||||||
|
1. Two columns on `tasks`: definition of done, and blocked-on. Blocked-on resolves
|
||||||
|
through Nexus like any other person reference, because identity lives in Nexus.
|
||||||
|
2. An edit path. Today a task is write-once except for its status, so the form has
|
||||||
|
nothing to save into.
|
||||||
|
3. `/tasks` grows the form: promote candidate to open only with a definition of done,
|
||||||
|
set a date, set blocked-on. A date set here writes a reminder.
|
||||||
|
4. A status-change grammar at stage 0, following `TaskCaptureGrammar`.
|
||||||
|
5. Counted stall shapes on `/tasks`, phrased as counts. No assessment.
|
||||||
|
|
||||||
|
Note for whoever picks up 3: `/tasks` accepts its POST without the step-up gate, while
|
||||||
|
`/routines` and `/tools` require a passkey. That was deliberate for capture. Adding an
|
||||||
|
edit path is the moment to re-argue it, not to inherit it silently.
|
||||||
|
|
||||||
|
Each is separable and each is worth stopping after.
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
# Plan: What the ambient calendar path should be
|
||||||
|
|
||||||
|
**The decision Vikunja #432 asked for. Written 04-08-2026.**
|
||||||
|
|
||||||
|
**Verdict: keep the endpoint, change the contract.** The relay app sends structured
|
||||||
|
fields, not a notification blob. The free-text parser stays as the degraded path, because
|
||||||
|
there is a real case where the phone cannot produce structure. Delete the endpoint only
|
||||||
|
if the answer to the one open question below is no.
|
||||||
|
|
||||||
|
## First, the task's premise is out of date
|
||||||
|
|
||||||
|
#432 states as confirmed that every ambient event lands on the day the notification was
|
||||||
|
posted, because there is no date parsing at all. That was true when the task was filed
|
||||||
|
and it is not true now.
|
||||||
|
|
||||||
|
`4e4c917` added `dayWords` and `dayOffset` (`internal/calendar/ambient.go:53`), so
|
||||||
|
"завтра в 15:00" now dates to tomorrow. The same commit added `ambientPastGrace`, which
|
||||||
|
refuses an event landing more than two hours before the notification, on the reasoning
|
||||||
|
that the day was inferred and a stale inference is wrong rather than late. `45a5e37`
|
||||||
|
(#482, this week) fixed a second dating bug the task did not know about: the wall clock
|
||||||
|
was resolved against the notification's own zone, so every ambient meeting on a non-UTC
|
||||||
|
box landed off by the deploy's UTC offset.
|
||||||
|
|
||||||
|
What is still missing is an explicit date. "5 августа в 15:00" and "12.08 15:00" carry no
|
||||||
|
day word, so they date to today and the past-grace check drops them. That is a smaller
|
||||||
|
defect than the one filed, and it fails safe rather than storing a wrong meeting.
|
||||||
|
|
||||||
|
The task's third question also has an answer, and the answer is yes. `internal/morning/plan.go:174`
|
||||||
|
prefixes an uncertain item with "похоже, " and `cmd/mavend/actions_query.go:334` carries
|
||||||
|
`Confidence < 1.0` into the calendar recital. Both readers hedge.
|
||||||
|
|
||||||
|
## The open question, and it is the owner's
|
||||||
|
|
||||||
|
**Can the phone read Android's calendar provider, or only the notification text?**
|
||||||
|
|
||||||
|
Everything follows from this and nothing in this repo can answer it.
|
||||||
|
|
||||||
|
A `NotificationListenerService` sees a title and a body. It cannot know a meeting's real
|
||||||
|
start, end or organiser, because those are not in the notification. So if the relay is
|
||||||
|
limited to the notification stream, free-text parsing on this side is not a choice, it is
|
||||||
|
the only thing available, and #432's suggestion that the phone send structured JSON
|
||||||
|
cannot be honoured.
|
||||||
|
|
||||||
|
If the app may instead read `CalendarContract`, it has the actual event rows, and the
|
||||||
|
whole parser stops being necessary. That is the better shape by a wide margin: a real
|
||||||
|
start and end, a real title, an explicit date, no clock-reading heuristic and no
|
||||||
|
past-grace guard, because nothing is inferred.
|
||||||
|
|
||||||
|
Reading the phone's calendar provider does not break the constraint the design was built
|
||||||
|
around. The refusal in `internal/calendar/ambient.go:10` is about holding a work
|
||||||
|
credential **on the homelab**, which is what ties the box's blast radius to the employer.
|
||||||
|
The phone already holds that session. Nothing new lands on homesrv either way.
|
||||||
|
|
||||||
|
**Assumption, and it needs his answer:** a managed work profile may block a third-party
|
||||||
|
app from reading work calendar rows. If it does, the notification stream is all there is.
|
||||||
|
|
||||||
|
## The decision
|
||||||
|
|
||||||
|
**Keep the endpoint.** Deleting it costs the parser, the tests and the wg-facing token,
|
||||||
|
and buys nothing while the question above is open. It is off unless `-ambient-token` is
|
||||||
|
set, so an unbuilt relay carries no surface today.
|
||||||
|
|
||||||
|
**Make structured the primary shape.** `/api/ambient` should accept an event with an
|
||||||
|
explicit start, end and title, and store it without parsing anything. Confidence stays
|
||||||
|
below 1.0 and the source stays `ambient:notif`, because the provenance claim is unchanged:
|
||||||
|
this is the phone telling Maven what it sees, not Maven reading a calendar.
|
||||||
|
|
||||||
|
**Keep the free-text shape as the degraded path.** It is what a notification-only relay
|
||||||
|
can send, and it is already written and tested.
|
||||||
|
|
||||||
|
**Delete it instead if** the relay is not going to be built. That is the one answer that
|
||||||
|
closes this without code, and it is his to give.
|
||||||
|
|
||||||
|
## What not to do
|
||||||
|
|
||||||
|
Do not add date parsing to the free-text path yet. That is the patch #432 explicitly
|
||||||
|
refuses to accept as closure, and it is the wrong order: if the relay can send a date, no
|
||||||
|
date parser is needed, and if it cannot, the parser is guessing at a date from text that
|
||||||
|
was never meant to carry one.
|
||||||
|
|
||||||
|
## Follow-on, unrelated to the decision
|
||||||
|
|
||||||
|
`cmd/mavweb/ambient.go:33` records a known gap worth keeping visible: an ambient meeting
|
||||||
|
writes `calendar_event_*` and never `calendar_busy`, so it is good enough to recite and
|
||||||
|
not good enough to suppress a nudge. That is backwards. Suppressing a nudge is the
|
||||||
|
lower-risk use of a low-confidence signal, and reciting one is the higher-risk use. It
|
||||||
|
needs an expiry on the busy level, so it is its own task either way.
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
# Plan: Where mavwaked and mavenclient run
|
||||||
|
|
||||||
|
**The decision Vikunja #463 asked for. Written 04-08-2026.**
|
||||||
|
|
||||||
|
**Verdict: not in compose on homesrv. They run on a client machine in the room he is in.**
|
||||||
|
The transport for that already exists and nothing needs building to allow it. What needs
|
||||||
|
building is a way to check the wake path at all, which is a separate task.
|
||||||
|
|
||||||
|
## The state that prompted this
|
||||||
|
|
||||||
|
`docker-compose.yml` runs mavend, mavsttd, mavttsd, mavweb and mavpoll. `mavwaked` and
|
||||||
|
`mavenclient` appear in no compose file and run as no host process. Both build under
|
||||||
|
`make build`. So the wake word and the voice-activity gate are untested by construction:
|
||||||
|
QA session 1 step 2 covers push-to-talk from `/dash` only, and #287 (voice session
|
||||||
|
quality) can never be more than half-answered while this holds.
|
||||||
|
|
||||||
|
## The reason is not hardware
|
||||||
|
|
||||||
|
homesrv has a microphone. It is a Lenovo IdeaPad 5 Pro, and `/proc/asound/cards` lists
|
||||||
|
the ACP digital mic array with capture devices at `/dev/snd/pcmC1D0c` and
|
||||||
|
`/dev/snd/pcmC2D0c`. Adding `/dev/snd` to compose and joining the `audio` group would
|
||||||
|
work.
|
||||||
|
|
||||||
|
It would also be pointless. A wake-word daemon is worth having in the room he is standing
|
||||||
|
in. homesrv is a server, so its microphone hears the room the server is in, which is not
|
||||||
|
where anybody talks to Maven. Wiring audio into a container to listen to an empty room is
|
||||||
|
work spent on a capability nobody can use.
|
||||||
|
|
||||||
|
## The reason it belongs off-box
|
||||||
|
|
||||||
|
`mavenclient` is a client by name and by design: microphone, then STT, then core, then
|
||||||
|
TTS. It is the one binary in the tree meant to run somewhere else. `mavwaked` is the gate
|
||||||
|
in front of it, so it goes wherever the microphone goes.
|
||||||
|
|
||||||
|
Maven already has components that are not containers on homesrv. `mavupdate` is a
|
||||||
|
host-side tool. The resident model, STT and TTS prefer the workstation and fall back to
|
||||||
|
homesrv (`docs/offload.md`). Off-box is a shape this system already has.
|
||||||
|
|
||||||
|
**And the wire already supports it.** `ipc.Dial` takes a netaddr seam address:
|
||||||
|
a bare path is the unix socket, and `tcp://host:port?token=...` reaches a core on another
|
||||||
|
host, with the token checked in `internal/netaddr` before `internal/ipc` sees the
|
||||||
|
connection. So a client machine reaching mavend over wg or the LAN needs no new protocol
|
||||||
|
work. It needs mavend to listen on TCP, which `deploy/mavend.json` does not currently ask
|
||||||
|
for.
|
||||||
|
|
||||||
|
## What this means for the QA plan
|
||||||
|
|
||||||
|
QA session 1 step 2 should say what it actually covers, which is push-to-talk through
|
||||||
|
`/dash`. It should not read as though it covers the voice loop. The wake path is checked
|
||||||
|
on the client machine or it is not checked, and today there is no client machine.
|
||||||
|
|
||||||
|
That is the honest state, and it is worse than the task suggests: this is not a
|
||||||
|
configuration gap that a compose entry closes. Until a machine with a microphone runs
|
||||||
|
`mavwaked` and `mavenclient` against a TCP-listening mavend, `internal/wake` and
|
||||||
|
`cmd/mavwaked`'s VAD are covered by their unit tests and by nothing else.
|
||||||
|
|
||||||
|
## What was wrong in CLAUDE.md
|
||||||
|
|
||||||
|
The daemon table lists all nine binaries with no column for where they run, which is how
|
||||||
|
this went unnoticed for as long as it did. It now says which two are not on the box.
|
||||||
+35
-9
@@ -1,6 +1,6 @@
|
|||||||
# QA plan: checking Maven properly
|
# QA plan: checking Maven properly
|
||||||
|
|
||||||
*Last verified: 2026-08-02 @ 20aa2d5. Living doc: correct it in place, do not append.*
|
*Last verified: 2026-08-04 @ 58635f1. Living doc: correct it in place, do not append.*
|
||||||
|
|
||||||
Written 2026-08-01, after the 35-PR stack landed and the box came back up.
|
Written 2026-08-01, after the 35-PR stack landed and the box came back up.
|
||||||
Refreshed 2026-08-02 against the live list, after PRs #85-#90.
|
Refreshed 2026-08-02 against the live list, after PRs #85-#90.
|
||||||
@@ -105,9 +105,9 @@ turns look misaligned when they are not.
|
|||||||
**Passes** (02-08-2026, five turns): `я рада`, `поняла`, `помогла`,
|
**Passes** (02-08-2026, five turns): `я рада`, `поняла`, `помогла`,
|
||||||
`проверила`, `записала`, `грустна`, `ты` throughout, no pet names.
|
`проверила`, `записала`, `грустна`, `ты` throughout, no pet names.
|
||||||
2. Press push-to-talk on `/dash`. Say `привет`. Confirm a spoken reply comes
|
2. Press push-to-talk on `/dash`. Say `привет`. Confirm a spoken reply comes
|
||||||
back. This is the only check that covers mic to STT to core to TTS to
|
back. This covers browser mic to STT to core to TTS as one path. It does
|
||||||
speaker as one path. It is also the path the eleven-day outage most likely
|
**not** cover the wake word or the voice-activity gate, and no step here
|
||||||
broke.
|
does — see below.
|
||||||
3. Say `тихий режим`. Expect `тихий режим включён. буду реже напоминать.` **Passes.**
|
3. Say `тихий режим`. Expect `тихий режим включён. буду реже напоминать.` **Passes.**
|
||||||
4. Say `выключи тихий режим`. Expect `тихий режим выключен.` Negation must win. **Passes.**
|
4. Say `выключи тихий режим`. Expect `тихий режим выключен.` Negation must win. **Passes.**
|
||||||
5. Say `в комнате тихо`. Quiet mode must NOT flip. Confirm on `/history` that no
|
5. Say `в комнате тихо`. Quiet mode must NOT flip. Confirm on `/history` that no
|
||||||
@@ -129,11 +129,18 @@ turns look misaligned when they are not.
|
|||||||
and stitch unrelated topics. Asked whether he should move flats, she opened
|
and stitch unrelated topics. Asked whether he should move flats, she opened
|
||||||
with the weather. That is 287, and it is a phrasing problem, not a loop problem.
|
with the weather. That is 287, and it is a phrasing problem, not a loop problem.
|
||||||
|
|
||||||
**The wake path cannot be checked as deployed.** `mavwaked` and `mavenclient`
|
**The wake path cannot be checked here, and that is now the decision rather
|
||||||
appear in no compose file and run as no host process. Step 2 covers only
|
than a gap.** `mavwaked` and `mavenclient` appear in no compose file and run as
|
||||||
push-to-talk, from `/dash` through mavsttd and mavttsd. Wake word and VAD
|
no host process. They are not going to. They belong on a client machine in the
|
||||||
are untested by construction. Decide whether they belong in compose or on a
|
room he is standing in, because homesrv's microphone is real and in the wrong
|
||||||
client machine, and say which in the deploy docs. Tracked as **463**.
|
room — **463**, written up in `docs/plans/17-where-the-voice-loop-runs.md`.
|
||||||
|
|
||||||
|
So the wake word and the VAD gate are covered by their unit tests and by
|
||||||
|
nothing else, and no session at this box changes that. Checking them needs a
|
||||||
|
machine with a microphone running both binaries against a TCP-listening mavend.
|
||||||
|
`ipc.Dial` already speaks `tcp://host:port?token=...`, so the work is a machine
|
||||||
|
and a config line, not protocol work. Until then, **287** can only be
|
||||||
|
half-answered, and step 2 above is push-to-talk, not the voice loop.
|
||||||
|
|
||||||
**319's single-token bug is fixed** (01-08-2026). Single-word Russian utterances no longer come
|
**319's single-token bug is fixed** (01-08-2026). Single-word Russian utterances no longer come
|
||||||
back as `не совсем поняла — можешь переформулировать?`. `привет` and `поужинал`
|
back as `не совсем поняла — можешь переформулировать?`. `привет` and `поужинал`
|
||||||
@@ -471,6 +478,25 @@ at any address (**478**). `allow_private` does work, measured both ways.
|
|||||||
`CaptureStart`. There is no `cmd/mavheard`, no mavweb route, and `mavenclient`
|
`CaptureStart`. There is no `cmd/mavheard`, no mavweb route, and `mavenclient`
|
||||||
never calls it (**480**). Two of its QA steps are also stale.
|
never calls it (**480**). Two of its QA steps are also stale.
|
||||||
|
|
||||||
|
**Four stale QA steps were rewritten on 04-08-2026** under **480**, against the
|
||||||
|
code rather than against what the plans said. All four failed the same way: the
|
||||||
|
daemon was right and the step described an older daemon.
|
||||||
|
|
||||||
|
| Step | Said | Says now |
|
||||||
|
|---|---|---|
|
||||||
|
| 253/3 | boots with the methods unknown | refuses to boot, `config.go:1651` |
|
||||||
|
| 253/10 | no `:transcript` note by default | true only with a summary present |
|
||||||
|
| 255/5 | `speaker: enrolment on, recognition BLOCKED` | that line is gone, the capability stays off |
|
||||||
|
| 252/3 | `vision: stored image <id-prefix>` | `vision: stored <id>`, `vision.go:199` |
|
||||||
|
|
||||||
|
Two of them are worth reading past the correction. 253/10 was false in exactly
|
||||||
|
the scenario 253/16 creates, because `writeNotes` saves the transcript whenever
|
||||||
|
the summary is empty so a dead llama-server does not lose the meeting. And 255/5
|
||||||
|
changed because `Recognizes()` was written as the gate, documented as one, and
|
||||||
|
never called — calling it turned `enabled` with no model from a half-working
|
||||||
|
capability into a refusal. Enrolling into a store nothing can match against is
|
||||||
|
not a working half.
|
||||||
|
|
||||||
**257, netscan.** Steps 2, 3 and 9 pass at unit level. Step 1 fails. Steps 4 to 8
|
**257, netscan.** Steps 2, 3 and 9 pass at unit level. Step 1 fails. Steps 4 to 8
|
||||||
need the block enabled. Step 10 is Bluetooth and stays skipped.
|
need the block enabled. Step 10 is Bluetooth and stays skipped.
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,295 @@
|
|||||||
|
package eval
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
_ "embed"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/kami/maven/internal/router"
|
||||||
|
)
|
||||||
|
|
||||||
|
//go:embed ru_ecosystem_v1.json
|
||||||
|
var reachFixtureJSON []byte
|
||||||
|
|
||||||
|
// Service — where an utterance arrives. Reach stops at the service boundary on
|
||||||
|
// purpose: whether Nexus knows the entity and whether Hexis holds a capability
|
||||||
|
// for it are those services' answers, and a fixture that asserted them would be
|
||||||
|
// measuring three systems and reporting one number.
|
||||||
|
type Service string
|
||||||
|
|
||||||
|
const (
|
||||||
|
// ServiceNone — the turn stays inside Maven.
|
||||||
|
ServiceNone Service = "none"
|
||||||
|
// ServicePraxis — handlePraxisAct dispatched to a capability.
|
||||||
|
ServicePraxis Service = "praxis"
|
||||||
|
// ServiceHexis — the text reached resolveEntityReference.
|
||||||
|
ServiceHexis Service = "hexis"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ReachCase — one utterance and the service it must arrive at.
|
||||||
|
//
|
||||||
|
// WantCapability is informational and unscored: it says which Praxis arm the fn
|
||||||
|
// slot should land on, so a failure reads as "reached Praxis, wrong arm" rather
|
||||||
|
// than only "reached Praxis". Scoring it would assert an alias table this
|
||||||
|
// package cannot import.
|
||||||
|
type ReachCase struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Utterance string `json:"utterance"`
|
||||||
|
Lang string `json:"lang"`
|
||||||
|
WantService Service `json:"want_service"`
|
||||||
|
WantCapability string `json:"want_capability"`
|
||||||
|
Tags []string `json:"tags"`
|
||||||
|
Note string `json:"note"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReachFixture — the versioned envelope, same shape as Fixture.
|
||||||
|
type ReachFixture struct {
|
||||||
|
SchemaVersion int `json:"schema_version"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
ReferenceNow string `json:"reference_now"`
|
||||||
|
Notes []string `json:"notes"`
|
||||||
|
Cases []ReachCase `json:"cases"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoadReach returns the embedded ecosystem fixture.
|
||||||
|
func LoadReach() (ReachFixture, error) {
|
||||||
|
var f ReachFixture
|
||||||
|
if err := json.Unmarshal(reachFixtureJSON, &f); err != nil {
|
||||||
|
return ReachFixture{}, fmt.Errorf("parse reach fixture: %w", err)
|
||||||
|
}
|
||||||
|
if f.SchemaVersion != SchemaVersion {
|
||||||
|
return ReachFixture{}, fmt.Errorf("reach fixture schema_version %d, want %d", f.SchemaVersion, SchemaVersion)
|
||||||
|
}
|
||||||
|
if len(f.Cases) == 0 {
|
||||||
|
return ReachFixture{}, fmt.Errorf("reach fixture has no cases")
|
||||||
|
}
|
||||||
|
return f, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Now — the fixture's reference clock, same contract as Fixture.Now.
|
||||||
|
func (f ReachFixture) Now() (time.Time, error) {
|
||||||
|
t, err := time.Parse(time.RFC3339, f.ReferenceNow)
|
||||||
|
if err != nil {
|
||||||
|
return time.Time{}, fmt.Errorf("parse reference_now %q: %w", f.ReferenceNow, err)
|
||||||
|
}
|
||||||
|
return t, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// PraxisAliases — the fn slots handlePraxisAct dispatches on, copied from
|
||||||
|
// praxisCapabilities in cmd/mavend/ecosystem_acts.go.
|
||||||
|
//
|
||||||
|
// It is a copy because the registry lives in package main and cannot be
|
||||||
|
// imported. That is a drift risk and it is deliberate: the alternative is
|
||||||
|
// lifting the whole capability registry out of the daemon, which is a
|
||||||
|
// refactor this measurement should not be carrying. TestPraxisAliasesShape
|
||||||
|
// asserts the arms are all present; a new capability adds a line here.
|
||||||
|
var PraxisAliases = map[string]string{
|
||||||
|
"list_attention": "list_attention",
|
||||||
|
"attention": "list_attention",
|
||||||
|
"внимание": "list_attention",
|
||||||
|
"что требует внимания": "list_attention",
|
||||||
|
"что нового": "list_attention",
|
||||||
|
"acknowledge_item": "acknowledge_item",
|
||||||
|
"принято": "acknowledge_item",
|
||||||
|
"понял": "acknowledge_item",
|
||||||
|
"поняла": "acknowledge_item",
|
||||||
|
"resolve_item": "resolve_item",
|
||||||
|
"сделано": "resolve_item",
|
||||||
|
"готово": "resolve_item",
|
||||||
|
"решено": "resolve_item",
|
||||||
|
"ignore_item": "ignore_item",
|
||||||
|
"игнорировать": "ignore_item",
|
||||||
|
"неважно": "ignore_item",
|
||||||
|
"pin_item": "pin_item",
|
||||||
|
"закрепить": "pin_item",
|
||||||
|
"list_changes": "list_changes",
|
||||||
|
"changes": "list_changes",
|
||||||
|
"изменения": "list_changes",
|
||||||
|
"что изменилось": "list_changes",
|
||||||
|
"entity_attention": "entity_attention",
|
||||||
|
"entity_status": "entity_attention",
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reach derives which ecosystem service a decision arrives at, assuming all
|
||||||
|
// three are configured. It mirrors actionAct in cmd/mavend/actions_act.go and
|
||||||
|
// hexisBeforeClarify in cmd/mavend/ecosystem_acts.go, in their order:
|
||||||
|
//
|
||||||
|
// 1. A clarified act with text and no fn reaches Hexis before the clarify
|
||||||
|
// question is ever asked. That path runs on the raw slots, so the matcher
|
||||||
|
// does not get to fill fn first.
|
||||||
|
// 2. Otherwise the act matcher may earn a fn from the text slot.
|
||||||
|
// 3. A fn that is a Praxis capability alias dispatches to Praxis.
|
||||||
|
// 4. Non-empty text reaches Hexis.
|
||||||
|
// 5. Anything else stays inside Maven.
|
||||||
|
//
|
||||||
|
// It returns the service and, for Praxis, the capability the fn landed on.
|
||||||
|
func Reach(d router.Decision, m router.ActMatcher) (Service, string) {
|
||||||
|
if d.Intent != router.IntentAct {
|
||||||
|
return ServiceNone, ""
|
||||||
|
}
|
||||||
|
if d.Clarify {
|
||||||
|
if !d.Slots.HasFn && d.Slots.Text != "" {
|
||||||
|
return ServiceHexis, ""
|
||||||
|
}
|
||||||
|
return ServiceNone, ""
|
||||||
|
}
|
||||||
|
fn, hasFn := d.Slots.Fn, d.Slots.HasFn
|
||||||
|
if !hasFn && d.Slots.Text != "" && m != nil {
|
||||||
|
if matched, _, ok := m.Match(d.Slots.Text); ok {
|
||||||
|
fn, hasFn = matched, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if hasFn {
|
||||||
|
if capability, ok := PraxisAliases[strings.ToLower(strings.TrimSpace(fn))]; ok {
|
||||||
|
return ServicePraxis, capability
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if d.Slots.Text != "" {
|
||||||
|
return ServiceHexis, ""
|
||||||
|
}
|
||||||
|
return ServiceNone, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReachOutcome — one scored case.
|
||||||
|
type ReachOutcome struct {
|
||||||
|
Case ReachCase
|
||||||
|
Decision router.Decision
|
||||||
|
Got Service
|
||||||
|
Capability string
|
||||||
|
Err error
|
||||||
|
Latency time.Duration
|
||||||
|
Pass bool
|
||||||
|
Reason string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReachReport — the aggregate.
|
||||||
|
//
|
||||||
|
// The two miss directions are kept apart because they cost different things.
|
||||||
|
// Missed is an utterance that should have reached a service and did not: he
|
||||||
|
// asks again, or does it himself. Overreach is an utterance that reached a
|
||||||
|
// service it had no business reaching, and on the Hexis side that is one
|
||||||
|
// resolution away from executing a capability nobody asked for.
|
||||||
|
type ReachReport struct {
|
||||||
|
Name string
|
||||||
|
Total int
|
||||||
|
Passed int
|
||||||
|
Missed int
|
||||||
|
Overreach int
|
||||||
|
// WrongService — reached a service, but the other one.
|
||||||
|
WrongService int
|
||||||
|
// WrongCapability — reached Praxis on the wrong arm. Reported, not failed.
|
||||||
|
WrongCapability int
|
||||||
|
Errors int
|
||||||
|
Outcomes []ReachOutcome
|
||||||
|
// ByService is keyed by the fixture's want_service.
|
||||||
|
ByService map[string]TagStat
|
||||||
|
ByTag map[string]TagStat
|
||||||
|
P50, P95 time.Duration
|
||||||
|
Max time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// Accuracy — fraction of cases that arrived where the fixture says they must.
|
||||||
|
func (r ReachReport) Accuracy() float64 {
|
||||||
|
if r.Total == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return float64(r.Passed) / float64(r.Total)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ScoreReach runs every case through the router and derives where it lands. It
|
||||||
|
// never fails the run on a route error: an erroring case scores as a miss and
|
||||||
|
// is counted, because "the model was down" and "the router was wrong" are
|
||||||
|
// different numbers.
|
||||||
|
func ScoreReach(ctx context.Context, name string, r Router, m router.ActMatcher, f ReachFixture) (ReachReport, error) {
|
||||||
|
now, err := f.Now()
|
||||||
|
if err != nil {
|
||||||
|
return ReachReport{}, err
|
||||||
|
}
|
||||||
|
rep := ReachReport{
|
||||||
|
Name: name,
|
||||||
|
Total: len(f.Cases),
|
||||||
|
ByService: map[string]TagStat{},
|
||||||
|
ByTag: map[string]TagStat{},
|
||||||
|
}
|
||||||
|
lat := make([]time.Duration, 0, len(f.Cases))
|
||||||
|
|
||||||
|
for _, c := range f.Cases {
|
||||||
|
start := time.Now()
|
||||||
|
d, err := r.Route(ctx, c.Utterance, now)
|
||||||
|
o := ReachOutcome{Case: c, Decision: d, Err: err, Latency: time.Since(start)}
|
||||||
|
lat = append(lat, o.Latency)
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
rep.Errors++
|
||||||
|
o.Reason = fmt.Sprintf("route error: %v", err)
|
||||||
|
default:
|
||||||
|
o.Got, o.Capability = Reach(d, m)
|
||||||
|
switch {
|
||||||
|
case o.Got == c.WantService:
|
||||||
|
o.Pass = true
|
||||||
|
if c.WantCapability != "" && o.Capability != c.WantCapability {
|
||||||
|
rep.WrongCapability++
|
||||||
|
o.Reason = fmt.Sprintf("reached praxis on %q, want %q", o.Capability, c.WantCapability)
|
||||||
|
}
|
||||||
|
case c.WantService == ServiceNone:
|
||||||
|
rep.Overreach++
|
||||||
|
o.Reason = fmt.Sprintf("reached %s, want none (intent %q, clarify %v, text %q)",
|
||||||
|
o.Got, d.Intent, d.Clarify, d.Slots.Text)
|
||||||
|
case o.Got == ServiceNone:
|
||||||
|
rep.Missed++
|
||||||
|
o.Reason = fmt.Sprintf("stayed local, want %s (intent %q, clarify %v, fn %q)",
|
||||||
|
c.WantService, d.Intent, d.Clarify, d.Slots.Fn)
|
||||||
|
default:
|
||||||
|
rep.WrongService++
|
||||||
|
o.Reason = fmt.Sprintf("reached %s, want %s (fn %q)", o.Got, c.WantService, d.Slots.Fn)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if o.Pass {
|
||||||
|
rep.Passed++
|
||||||
|
}
|
||||||
|
bump(rep.ByService, string(c.WantService), o.Pass)
|
||||||
|
for _, tag := range c.Tags {
|
||||||
|
bump(rep.ByTag, tag, o.Pass)
|
||||||
|
}
|
||||||
|
rep.Outcomes = append(rep.Outcomes, o)
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Slice(lat, func(i, j int) bool { return lat[i] < lat[j] })
|
||||||
|
rep.P50, rep.P95 = percentile(lat, 0.50), percentile(lat, 0.95)
|
||||||
|
if len(lat) > 0 {
|
||||||
|
rep.Max = lat[len(lat)-1]
|
||||||
|
}
|
||||||
|
return rep, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// String renders the report.
|
||||||
|
func (r ReachReport) String() string {
|
||||||
|
var b strings.Builder
|
||||||
|
fmt.Fprintf(&b, "%s: %d/%d reached the right place (%.1f%%)\n", r.Name, r.Passed, r.Total, 100*r.Accuracy())
|
||||||
|
fmt.Fprintf(&b, " missed: %d (should have reached, didn't) | overreach: %d (reached, shouldn't) | wrong service: %d | wrong praxis arm: %d | errors: %d\n",
|
||||||
|
r.Missed, r.Overreach, r.WrongService, r.WrongCapability, r.Errors)
|
||||||
|
fmt.Fprintf(&b, " latency: p50 %s p95 %s max %s\n", r.P50, r.P95, r.Max)
|
||||||
|
fmt.Fprintf(&b, " by want: %s\n", renderStats(r.ByService))
|
||||||
|
fmt.Fprintf(&b, " by tag: %s\n", renderStats(r.ByTag))
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Failures — the per-case detail, sorted by ID so two runs diff cleanly. A
|
||||||
|
// passing case with a wrong Praxis arm is listed too: it carries a reason.
|
||||||
|
func (r ReachReport) Failures() string {
|
||||||
|
var b strings.Builder
|
||||||
|
out := append([]ReachOutcome(nil), r.Outcomes...)
|
||||||
|
sort.Slice(out, func(i, j int) bool { return out[i].Case.ID < out[j].Case.ID })
|
||||||
|
for _, o := range out {
|
||||||
|
if o.Reason == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, " %s %q: %s\n", o.Case.ID, o.Case.Utterance, o.Reason)
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
@@ -0,0 +1,228 @@
|
|||||||
|
package eval
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/kami/maven/internal/router"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLoadReachFixture(t *testing.T) {
|
||||||
|
f, err := LoadReach()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadReach: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := f.Now(); err != nil {
|
||||||
|
t.Fatalf("Now: %v", err)
|
||||||
|
}
|
||||||
|
valid := map[Service]bool{ServiceNone: true, ServicePraxis: true, ServiceHexis: true}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
byService := map[Service]int{}
|
||||||
|
for _, c := range f.Cases {
|
||||||
|
if c.ID == "" || seen[c.ID] {
|
||||||
|
t.Errorf("case %q: empty or duplicate id", c.ID)
|
||||||
|
}
|
||||||
|
seen[c.ID] = true
|
||||||
|
if !valid[c.WantService] {
|
||||||
|
t.Errorf("%s: want_service %q, want none|praxis|hexis", c.ID, c.WantService)
|
||||||
|
}
|
||||||
|
if c.Lang != "ru" && c.Lang != "en" {
|
||||||
|
t.Errorf("%s: lang %q, want ru|en", c.ID, c.Lang)
|
||||||
|
}
|
||||||
|
// A capability only means something on the Praxis side, and it must
|
||||||
|
// name an arm that exists — otherwise the case asserts a target the
|
||||||
|
// dispatch can never hit.
|
||||||
|
if c.WantCapability != "" {
|
||||||
|
if c.WantService != ServicePraxis {
|
||||||
|
t.Errorf("%s: want_capability on %q", c.ID, c.WantService)
|
||||||
|
}
|
||||||
|
if PraxisAliases[c.WantCapability] != c.WantCapability {
|
||||||
|
t.Errorf("%s: capability %q is not a praxis arm", c.ID, c.WantCapability)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
byService[c.WantService]++
|
||||||
|
}
|
||||||
|
// Coverage floor. The negative cases matter most: without them a router
|
||||||
|
// that sent everything to Hexis would score perfectly.
|
||||||
|
for s := range valid {
|
||||||
|
if byService[s] < 5 {
|
||||||
|
t.Errorf("want_service %q has %d cases, want >= 5", s, byService[s])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestReachFixtureIsHeldOut — same rule as TestFixtureIsHeldOut. The classifier
|
||||||
|
// routes by similarity to frozen seeds, so a case copied from models/seeds
|
||||||
|
// would measure memorisation rather than reach.
|
||||||
|
func TestReachFixtureIsHeldOut(t *testing.T) {
|
||||||
|
f, err := LoadReach()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadReach: %v", err)
|
||||||
|
}
|
||||||
|
seeds := loadSeeds(t)
|
||||||
|
for _, c := range f.Cases {
|
||||||
|
if src, ok := seeds[normalize(c.Utterance)]; ok {
|
||||||
|
t.Errorf("%s: %q is verbatim in %s — not held out", c.ID, c.Utterance, src)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestReachDerivation pins the gate order that ScoreReach depends on. These are
|
||||||
|
// the four branches of actionAct plus the clarify pre-empt, asserted directly
|
||||||
|
// so a change to cmd/mavend/actions_act.go that this package no longer mirrors
|
||||||
|
// fails here rather than silently moving the score.
|
||||||
|
func TestReachDerivation(t *testing.T) {
|
||||||
|
m := router.DefaultActMatcher{Fns: actFns}
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
dec router.Decision
|
||||||
|
want Service
|
||||||
|
capability string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "act with a praxis alias in the fn slot",
|
||||||
|
dec: router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: "list_attention", HasFn: true}},
|
||||||
|
want: ServicePraxis, capability: "list_attention",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "act with a russian praxis alias",
|
||||||
|
dec: router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: "готово", HasFn: true}},
|
||||||
|
want: ServicePraxis, capability: "resolve_item",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "act with an entity but no fn earns one from the matcher",
|
||||||
|
dec: router.Decision{Intent: router.IntentAct, Slots: router.Slots{Text: "выключи свет в спальне"}},
|
||||||
|
want: ServiceHexis,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "act with a non-praxis fn and text goes to hexis",
|
||||||
|
dec: router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: "restart", HasFn: true, Text: "перезапусти гитею"}},
|
||||||
|
want: ServiceHexis,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "act with a fn and no text stays local",
|
||||||
|
dec: router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: "restart", HasFn: true}},
|
||||||
|
want: ServiceNone,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "a clarified act with text still reaches hexis",
|
||||||
|
dec: router.Decision{Intent: router.IntentAct, Clarify: true, Slots: router.Slots{Text: "выключи это"}},
|
||||||
|
want: ServiceHexis,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "a clarified act that already has a fn does not",
|
||||||
|
dec: router.Decision{Intent: router.IntentAct, Clarify: true, Slots: router.Slots{Fn: "restart", HasFn: true, Text: "перезапусти"}},
|
||||||
|
want: ServiceNone,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no intent but act reaches nothing",
|
||||||
|
dec: router.Decision{Intent: router.IntentQuery, Slots: router.Slots{Fn: "list_attention", HasFn: true, Text: "что требует внимания"}},
|
||||||
|
want: ServiceNone,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got, capability := Reach(tc.dec, m)
|
||||||
|
if got != tc.want {
|
||||||
|
t.Errorf("Reach = %q, want %q", got, tc.want)
|
||||||
|
}
|
||||||
|
if capability != tc.capability {
|
||||||
|
t.Errorf("capability = %q, want %q", capability, tc.capability)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPraxisAliasesShape — the copy of praxisCapabilities in reach.go is a
|
||||||
|
// drift risk (see its comment). This does not close it, but it does catch the
|
||||||
|
// cheap half: an arm losing all its aliases, or an alias pointing at an arm
|
||||||
|
// that no longer has a canonical name.
|
||||||
|
func TestPraxisAliasesShape(t *testing.T) {
|
||||||
|
arms := map[string]int{}
|
||||||
|
for alias, capability := range PraxisAliases {
|
||||||
|
if alias == "" || capability == "" {
|
||||||
|
t.Errorf("empty alias or capability: %q → %q", alias, capability)
|
||||||
|
}
|
||||||
|
arms[capability]++
|
||||||
|
}
|
||||||
|
for _, want := range []string{
|
||||||
|
"list_attention", "acknowledge_item", "resolve_item",
|
||||||
|
"ignore_item", "pin_item", "list_changes", "entity_attention",
|
||||||
|
} {
|
||||||
|
if arms[want] == 0 {
|
||||||
|
t.Errorf("praxis arm %q has no aliases", want)
|
||||||
|
}
|
||||||
|
// Every arm must be reachable by its own name, which is the alias the
|
||||||
|
// LLM router's fn slot actually emits.
|
||||||
|
if PraxisAliases[want] != want {
|
||||||
|
t.Errorf("arm %q does not alias to itself", want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestReachBaselineHash — the deterministic ratchet. Same deal as
|
||||||
|
// TestClassifierBaseline: the hash embedder never clears the confidence gate on
|
||||||
|
// paraphrases, so almost everything lands on clarify, and the number this
|
||||||
|
// asserts is the overreach count rather than the accuracy.
|
||||||
|
//
|
||||||
|
// Overreach is the direction worth a hard assertion. An utterance that should
|
||||||
|
// stay inside Maven and instead resolves an entity through Nexus is one Hexis
|
||||||
|
// capability away from executing something nobody asked for, and unlike a miss
|
||||||
|
// he never gets asked about it.
|
||||||
|
func TestReachBaselineHash(t *testing.T) {
|
||||||
|
f, err := LoadReach()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadReach: %v", err)
|
||||||
|
}
|
||||||
|
m := router.DefaultActMatcher{Fns: actFns}
|
||||||
|
rep, err := ScoreReach(context.Background(), "reach: classifier+hash",
|
||||||
|
newBaselineRouter(t, router.NewHashEmbedder(1024), nil), m, f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ScoreReach: %v", err)
|
||||||
|
}
|
||||||
|
t.Log("\n" + rep.String() + rep.Failures())
|
||||||
|
|
||||||
|
if rep.Overreach > 4 {
|
||||||
|
t.Errorf("%d utterances reached a service they should not have, want <= 4:\n%s",
|
||||||
|
rep.Overreach, rep.Failures())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestReachBaselineONNX — the deployed configuration: the cascade with the
|
||||||
|
// multilingual e5 embedder. Opt-in via MAVEN_ONNX_LIB, same as TestONNXBaseline,
|
||||||
|
// because deps/ is gitignored. `make eval-reach` points it at the vendored copy.
|
||||||
|
//
|
||||||
|
// Reports rather than asserts. This is the number Vikunja #405 asked for, and a
|
||||||
|
// threshold invented alongside the first measurement is not a ratchet, it is a
|
||||||
|
// guess written down twice.
|
||||||
|
func TestReachBaselineONNX(t *testing.T) {
|
||||||
|
lib := os.Getenv("MAVEN_ONNX_LIB")
|
||||||
|
if lib == "" {
|
||||||
|
t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
|
||||||
|
}
|
||||||
|
model := filepath.Join("../../..", "models/embedder/multilingual-e5-small/model_quantized.onnx")
|
||||||
|
tok := filepath.Join("../../..", "models/embedder/multilingual-e5-small/tokenizer.json")
|
||||||
|
for _, p := range []string{lib, model, tok} {
|
||||||
|
if _, err := os.Stat(p); err != nil {
|
||||||
|
t.Skipf("missing %s: %v", p, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
emb, err := router.NewONNXEmbedder(model, tok, lib)
|
||||||
|
if err != nil {
|
||||||
|
t.Skipf("onnx embedder unavailable: %v", err)
|
||||||
|
}
|
||||||
|
defer emb.Close()
|
||||||
|
|
||||||
|
f, err := LoadReach()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadReach: %v", err)
|
||||||
|
}
|
||||||
|
rep, err := ScoreReach(context.Background(), "reach: classifier+onnx",
|
||||||
|
newBaselineRouter(t, emb, nil), router.DefaultActMatcher{Fns: actFns}, f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ScoreReach: %v", err)
|
||||||
|
}
|
||||||
|
t.Log("\n" + rep.String() + rep.Failures())
|
||||||
|
}
|
||||||
@@ -0,0 +1,284 @@
|
|||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"name": "ru_ecosystem_v1",
|
||||||
|
"reference_now": "2026-08-04T10:00:00+03:00",
|
||||||
|
"notes": [
|
||||||
|
"Held out from models/seeds/*.txt, same rule as ru_routing_v1: a case that is verbatim a seed measures memorisation.",
|
||||||
|
"want_service is where the utterance must ARRIVE, not what it must achieve. Praxis means handlePraxisAct dispatched to a capability; hexis means the text reached resolveEntityReference. Whether the entity exists in Nexus and the capability exists in Hexis is the services' answer, not the router's.",
|
||||||
|
"The reach model is derived in reach.go from cmd/mavend/actions_act.go. Praxis needs IntentAct AND a fn slot whose value is a praxis capability alias. Hexis needs IntentAct AND non-empty text, and it also fires from hexisBeforeClarify, so a clarified act still reaches it.",
|
||||||
|
"want_capability is informational and is not scored. It records which praxis arm the fn should land on, so a report can say WHERE a praxis reach went wrong rather than only that it did.",
|
||||||
|
"none means the utterance must NOT reach either service. Those cases are the expensive direction: a note or a query that lands on a mutating Hexis path is a wrong act, not a missed one."
|
||||||
|
],
|
||||||
|
"cases": [
|
||||||
|
{
|
||||||
|
"id": "eco-ru-001",
|
||||||
|
"utterance": "выключи свет в спальне",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "hexis",
|
||||||
|
"want_capability": "",
|
||||||
|
"tags": ["home", "mutating"],
|
||||||
|
"note": "The canonical act. Verb in the allowlist, entity text after it."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-002",
|
||||||
|
"utterance": "включи свет на кухне, пожалуйста",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "hexis",
|
||||||
|
"want_capability": "",
|
||||||
|
"tags": ["home", "mutating"],
|
||||||
|
"note": ""
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-003",
|
||||||
|
"utterance": "перезапусти вукунью",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "hexis",
|
||||||
|
"want_capability": "",
|
||||||
|
"tags": ["homelab", "mutating"],
|
||||||
|
"note": "Service name in Russian, declined. Nexus owns the alias, the router only has to hand it over."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-004",
|
||||||
|
"utterance": "перезагрузи прокси",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "hexis",
|
||||||
|
"want_capability": "",
|
||||||
|
"tags": ["homelab", "mutating"],
|
||||||
|
"note": ""
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-005",
|
||||||
|
"utterance": "останови гитею на минуту",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "hexis",
|
||||||
|
"want_capability": "",
|
||||||
|
"tags": ["homelab", "mutating"],
|
||||||
|
"note": "Trailing qualifier the entity resolver has to survive."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-006",
|
||||||
|
"utterance": "запусти бэкап на нексусе",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "hexis",
|
||||||
|
"want_capability": "",
|
||||||
|
"tags": ["homelab", "mutating"],
|
||||||
|
"note": ""
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-007",
|
||||||
|
"utterance": "закрой шторы в комнате",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "hexis",
|
||||||
|
"want_capability": "",
|
||||||
|
"tags": ["home", "mutating"],
|
||||||
|
"note": ""
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-008",
|
||||||
|
"utterance": "открой гараж",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "hexis",
|
||||||
|
"want_capability": "",
|
||||||
|
"tags": ["home", "mutating"],
|
||||||
|
"note": "Two words. The single-token thinner spares it, the verb ending carries the subject."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-009",
|
||||||
|
"utterance": "поставь чайник",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "hexis",
|
||||||
|
"want_capability": "",
|
||||||
|
"tags": ["home", "mutating"],
|
||||||
|
"note": ""
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-010",
|
||||||
|
"utterance": "выключи телевизор в гостиной",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "hexis",
|
||||||
|
"want_capability": "",
|
||||||
|
"tags": ["home", "mutating"],
|
||||||
|
"note": ""
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-011",
|
||||||
|
"utterance": "что требует внимания",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "praxis",
|
||||||
|
"want_capability": "list_attention",
|
||||||
|
"tags": ["attention", "reading"],
|
||||||
|
"note": "The task's own example. Query-shaped, and the reach model only dispatches to Praxis from IntentAct, so this is the case most likely to miss."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-012",
|
||||||
|
"utterance": "что сейчас требует внимания",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "praxis",
|
||||||
|
"want_capability": "list_attention",
|
||||||
|
"tags": ["attention", "reading"],
|
||||||
|
"note": ""
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-013",
|
||||||
|
"utterance": "что нового по проектам",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "praxis",
|
||||||
|
"want_capability": "list_attention",
|
||||||
|
"tags": ["attention", "reading"],
|
||||||
|
"note": ""
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-014",
|
||||||
|
"utterance": "что изменилось за сегодня",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "praxis",
|
||||||
|
"want_capability": "list_changes",
|
||||||
|
"tags": ["attention", "reading"],
|
||||||
|
"note": ""
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-015",
|
||||||
|
"utterance": "покажи изменения",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "praxis",
|
||||||
|
"want_capability": "list_changes",
|
||||||
|
"tags": ["attention", "reading"],
|
||||||
|
"note": ""
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-016",
|
||||||
|
"utterance": "отметь это как сделанное",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "praxis",
|
||||||
|
"want_capability": "resolve_item",
|
||||||
|
"tags": ["lifecycle", "mutating"],
|
||||||
|
"note": "The task's second example. The item id is the value slot and is missing here, so the capability asks for it — reaching Praxis with no id is still reaching Praxis."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-017",
|
||||||
|
"utterance": "готово, закрывай",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "praxis",
|
||||||
|
"want_capability": "resolve_item",
|
||||||
|
"tags": ["lifecycle", "mutating"],
|
||||||
|
"note": ""
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-018",
|
||||||
|
"utterance": "принято, я это видел",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "praxis",
|
||||||
|
"want_capability": "acknowledge_item",
|
||||||
|
"tags": ["lifecycle", "mutating"],
|
||||||
|
"note": "Acknowledged is not resolved. The alias must not collapse into the resolve arm."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-019",
|
||||||
|
"utterance": "игнорировать это пока",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "praxis",
|
||||||
|
"want_capability": "ignore_item",
|
||||||
|
"tags": ["lifecycle", "mutating"],
|
||||||
|
"note": ""
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-020",
|
||||||
|
"utterance": "закрепи этот пункт",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "praxis",
|
||||||
|
"want_capability": "pin_item",
|
||||||
|
"tags": ["lifecycle", "mutating"],
|
||||||
|
"note": ""
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-021",
|
||||||
|
"utterance": "что там с нексусом",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "praxis",
|
||||||
|
"want_capability": "entity_attention",
|
||||||
|
"tags": ["attention", "reading"],
|
||||||
|
"note": "Scoped attention. The alias list for this arm is grammar names only, so free Russian reaching it depends on a grammar that does not exist yet."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-022",
|
||||||
|
"utterance": "как дела у праксиса",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "praxis",
|
||||||
|
"want_capability": "entity_attention",
|
||||||
|
"tags": ["attention", "reading"],
|
||||||
|
"note": ""
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-023",
|
||||||
|
"utterance": "напомни завтра в девять позвонить маме",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "none",
|
||||||
|
"want_capability": "",
|
||||||
|
"tags": ["negative"],
|
||||||
|
"note": "A reminder is Maven's own store. Nothing outside her hears it."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-024",
|
||||||
|
"utterance": "запиши что мне понравился этот подход",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "none",
|
||||||
|
"want_capability": "",
|
||||||
|
"tags": ["negative"],
|
||||||
|
"note": "A note carrying a verb-shaped word. It must not become an act."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-025",
|
||||||
|
"utterance": "какая завтра погода",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "none",
|
||||||
|
"want_capability": "",
|
||||||
|
"tags": ["negative"],
|
||||||
|
"note": ""
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-026",
|
||||||
|
"utterance": "мой рост сто восемьдесят два",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "none",
|
||||||
|
"want_capability": "",
|
||||||
|
"tags": ["negative"],
|
||||||
|
"note": "A fact about him. The personal boundary means this never leaves the box."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-027",
|
||||||
|
"utterance": "как ты сегодня",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "none",
|
||||||
|
"want_capability": "",
|
||||||
|
"tags": ["negative"],
|
||||||
|
"note": ""
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-028",
|
||||||
|
"utterance": "выключи",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "none",
|
||||||
|
"want_capability": "",
|
||||||
|
"tags": ["negative", "ambiguous"],
|
||||||
|
"note": "A verb with no target. Reaching Hexis with an empty entity text is the failure this case guards: it must clarify, not resolve."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-029",
|
||||||
|
"utterance": "сделай это",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "none",
|
||||||
|
"want_capability": "",
|
||||||
|
"tags": ["negative", "ambiguous"],
|
||||||
|
"note": "The known dangerous case. It carries text, so under the current gates it does reach Hexis — the fixture says it should not, and that disagreement is the finding."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "eco-ru-030",
|
||||||
|
"utterance": "во сколько у меня встреча",
|
||||||
|
"lang": "ru",
|
||||||
|
"want_service": "none",
|
||||||
|
"want_capability": "",
|
||||||
|
"tags": ["negative"],
|
||||||
|
"note": "Stage 0 routes agenda questions to query. The calendar is Maven's, not Praxis's."
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user