Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| a0293bac85 | |||
| c1d9a4547b | |||
| 6499f6365e | |||
| 1b3af05d0a | |||
| 1f8e9f21ce | |||
| e7537d032e | |||
| 2b3e34c7e8 | |||
| b86172a98d | |||
| 4f6dec0cf2 | |||
| 23ad5c0247 |
@@ -0,0 +1,113 @@
|
||||
// Command labelgen labels utterances with the stage 0 grammars and prints JSONL.
|
||||
//
|
||||
// docs/plans/18-routing-heads-on-e5-small.md calls the labeled set the whole
|
||||
// project, and it names the stage 0 grammars as the high-precision label
|
||||
// functions to start from. This runs them — the real ones, in the real
|
||||
// buildRouter order — rather than a reimplementation, so a rule change moves
|
||||
// the training data with it.
|
||||
//
|
||||
// A grammar that declines leaves the line unlabeled. Those go to the model, and
|
||||
// keeping them is the point: a set labeled only by the rules teaches only the
|
||||
// rules.
|
||||
//
|
||||
// go run ./cmd/labelgen < utterances.txt > labeled.jsonl
|
||||
//
|
||||
// The wakeword-act grammar is absent, because its allowlist is the deployment's
|
||||
// enabled tool names and this tool has no deployment. Every other rule is here.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/kami/maven/internal/router"
|
||||
)
|
||||
|
||||
// label is one output row. The grammar name rides along so a reviewer can see
|
||||
// which rule made the claim, and so a rule that turns out to be wrong can have
|
||||
// its rows pulled without re-running everything.
|
||||
type label struct {
|
||||
Utterance string `json:"utterance"`
|
||||
Intent string `json:"intent,omitempty"`
|
||||
Grammar string `json:"grammar,omitempty"`
|
||||
Key string `json:"key,omitempty"`
|
||||
Value string `json:"value,omitempty"`
|
||||
Fn string `json:"fn,omitempty"`
|
||||
Text string `json:"text,omitempty"`
|
||||
Labeled bool `json:"labeled"`
|
||||
}
|
||||
|
||||
// grammars mirrors buildRouter's order in cmd/mavend/voicewire.go. Order is
|
||||
// load-bearing there and so it is here: the agenda rules must sit after the
|
||||
// clock rules, Praxis before the capture marker, the narrative rules last.
|
||||
func grammars() []router.Grammar {
|
||||
var g []router.Grammar
|
||||
g = append(g, router.SystemTimeDateGrammars()...)
|
||||
g = append(g, router.AgendaQueryGrammars()...)
|
||||
g = append(g, router.FeedQueryGrammar())
|
||||
g = append(g, router.TaskListGrammar())
|
||||
g = append(g, router.ListGrammars()...)
|
||||
g = append(g, router.ReminderGrammar())
|
||||
g = append(g, router.PraxisGrammars()...)
|
||||
g = append(g, router.TaskCaptureGrammar())
|
||||
g = append(g, router.NarrativeQueryGrammars()...)
|
||||
return g
|
||||
}
|
||||
|
||||
func match(gs []router.Grammar, utterance string) label {
|
||||
out := label{Utterance: utterance}
|
||||
for _, g := range gs {
|
||||
m := g.Pattern.FindStringSubmatch(utterance)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
d, ok := g.Build(m)
|
||||
if !ok {
|
||||
continue // the rule saw its shape and declined it
|
||||
}
|
||||
out.Intent = string(d.Intent)
|
||||
out.Grammar = g.Name
|
||||
out.Key = d.Slots.Key
|
||||
out.Value = d.Slots.Value
|
||||
out.Fn = d.Slots.Fn
|
||||
out.Text = d.Slots.Text
|
||||
out.Labeled = true
|
||||
return out
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func main() {
|
||||
gs := grammars()
|
||||
in := bufio.NewScanner(os.Stdin)
|
||||
in.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
out := bufio.NewWriter(os.Stdout)
|
||||
defer out.Flush()
|
||||
|
||||
enc := json.NewEncoder(out)
|
||||
var seen, labeled int
|
||||
for in.Scan() {
|
||||
line := strings.TrimSpace(in.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
seen++
|
||||
l := match(gs, line)
|
||||
if l.Labeled {
|
||||
labeled++
|
||||
}
|
||||
if err := enc.Encode(l); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "labelgen:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := in.Err(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "labelgen:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
// Coverage on stderr, so the count is visible without polluting the JSONL.
|
||||
fmt.Fprintf(os.Stderr, "labelgen: %d/%d labeled by %d grammars\n", labeled, seen, len(gs))
|
||||
}
|
||||
@@ -139,9 +139,9 @@ func (h *reactiveHandler) handlePraxisAct(ctx context.Context, dec router.Decisi
|
||||
// praxisItemAction is the shared shape of the item-lifecycle capabilities: take
|
||||
// an item id from the value slot, call one Praxis endpoint, trace the result.
|
||||
type praxisItemAction struct {
|
||||
verbs []string
|
||||
ask string // reply when no item id was given
|
||||
op string // trace + log name of the operation
|
||||
verbs []string
|
||||
ask string // reply when no item id was given
|
||||
op string // trace + log name of the operation
|
||||
// failure is the first half of the reply when the Praxis call errors: which
|
||||
// operation did not happen. ecosystemGap supplies the second half, which
|
||||
// names Praxis and splits a refused token from an outage — those two used to
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
# Alarm verbs reach stage 0
|
||||
|
||||
**06-08-2026. V-627.** Measured with `TestONNXBaseline`, 91-case RU routing fixture,
|
||||
classifier plus the ONNX embedder. No LLM arm in this run.
|
||||
|
||||
## What was wrong
|
||||
|
||||
`lexicon.ReminderVerbs` held five words and none of them named an alarm. `ReminderGrammar`
|
||||
in `internal/router/stage0.go` did not read the set at all: it carried the literal
|
||||
`напомни|remind me`. So no part of the cascade recognised `разбуди`.
|
||||
|
||||
Three fixture cases ride on that. Under the classifier they went to fact and act at high
|
||||
confidence, so the failure was never a near miss:
|
||||
|
||||
- `ru-rem-005` "разбуди меня в 6:30" to fact at 0.918
|
||||
- `ru-rem-009` "разбуди меня полвосьмого" to act at 0.941
|
||||
- `en-rem-002` "wake me at 6:15" to fact at 0.899
|
||||
|
||||
Found while training the V-546 intent head, where the same three cases went to system. The
|
||||
head reads a spoken time with no known verb in front of it as a clock question. The
|
||||
classifier was making the same mistake in its own way.
|
||||
|
||||
## The change
|
||||
|
||||
Four alarm imperatives and bare `wake` join `reminder_verbs`. `ReminderGrammar` builds its
|
||||
alternation from the set, longest alternative first, and eats an optional `мне`, `меня` or
|
||||
`me` before the body.
|
||||
|
||||
Longest-first is load-bearing. Go's alternation is leftmost-first rather than longest-match,
|
||||
so `напомнить` listed after `напомни` would never match.
|
||||
|
||||
## Result
|
||||
|
||||
**66/91 to 69/91, 72.5% to 75.8% full.** Three cases gained, none lost.
|
||||
|
||||
All three are the alarms above, and each now carries its time slot, which it did not before.
|
||||
Clarify counts unchanged at 0 false and 8 missed. The two remaining system failures,
|
||||
`какое число завтра` and `какой день недели послезавтра`, failed at baseline too.
|
||||
|
||||
## What this does not fix
|
||||
|
||||
The lexicon addition on its own moved nothing. Measured before touching the grammar:
|
||||
**66/91**, exactly the baseline. Every consumer of `reminder_verbs` reads it after a reminder
|
||||
route already exists. A verb that cannot win the route is a verb nobody asks about. The
|
||||
grammar was the whole change.
|
||||
|
||||
Lemma matching in `isReminderVerb` now covers `разбудил` as well as `разбуди`, because one
|
||||
lemma holds both. That is the trap `cmd/mavend/quiet_toggle.go` documents for `говори`. It
|
||||
is tolerable here and not in the quiet toggle. `isReminderVerb` runs only on an utterance
|
||||
already routed to reminder, and it decides where the subject starts. A quiet match flips a
|
||||
daemon-wide setting from any channel.
|
||||
@@ -0,0 +1,82 @@
|
||||
# Gemma as a label function, and what it found in the seeds
|
||||
|
||||
**06-08-2026. V-546.** Measured on workpc against gemma-4-12b-it-qat-UD-Q4_K_XL.
|
||||
|
||||
`docs/plans/18-routing-heads-on-e5-small.md` puts the labeled set at 20k examples through
|
||||
gemma, costing 2 to 4 hours of the card. This is the check before spending that. Gemma
|
||||
labels the 344 hand-written classifier seeds. Agreement with the label a person already
|
||||
chose is a precision number rather than a guess.
|
||||
|
||||
## What ran
|
||||
|
||||
`cmd/labelgen` runs the stage 0 grammars. The real ones, in `buildRouter` order, minus
|
||||
`wakeword-act`, whose allowlist is a deployment's enabled tool names. It labels 62 of 339
|
||||
seed lines and leaves the rest.
|
||||
|
||||
The remaining 277 went to gemma through the daemon's own `routeSystem` prompt and
|
||||
`routeGrammar`, both extracted from `internal/router/llmrouter.go` at run time rather than
|
||||
retyped. Temperature 0.
|
||||
|
||||
## Cost
|
||||
|
||||
**334ms per call, 0 unparsed of 277.** The GBNF held every time. At that rate the plan's
|
||||
20k examples is under two hours of card, which matches its estimate.
|
||||
|
||||
## The stage 0 rules as label functions
|
||||
|
||||
Agreement between the grammar's label and the seed file the line came from:
|
||||
|
||||
| seed intent | agree |
|
||||
|---|---|
|
||||
| reminder | 37/37 |
|
||||
| query | 9/10 |
|
||||
| system | 7/8 |
|
||||
| act | 2/2 |
|
||||
| chat | 0/4 |
|
||||
| note | 0/1 |
|
||||
|
||||
`ReminderGrammar` at 37/37 is the evidence the plan wanted. The chat column is a defect
|
||||
rather than a disagreement: `chatNarrativeTopics` is Russian-only, so `tell me about
|
||||
yourself` survives the decline and routes IntentQuery with topic `yourself`. Filed as
|
||||
V-625, which also records that `как дела у сервера` appears verbatim in two seed files
|
||||
under two intents.
|
||||
|
||||
## Gemma against the seeds
|
||||
|
||||
**197/277, 71.1%.** By intent:
|
||||
|
||||
| seed intent | agree |
|
||||
|---|---|
|
||||
| note | 33/33 |
|
||||
| act | 57/64 |
|
||||
| fact | 37/40 |
|
||||
| query | 51/54 |
|
||||
| chat | 15/35 |
|
||||
| system | 4/43 |
|
||||
| reminder | 0/8 |
|
||||
|
||||
The number is not gemma's error rate. Reading the 80 disagreements, most are the seed files
|
||||
and the prompt holding different definitions of the same intent. Three boundaries carry 42
|
||||
of them, and V-626 is the fix:
|
||||
|
||||
- **system, 26 lines.** The prompt restricts system to the clock, the calendar date and the
|
||||
assistant itself. The seeds also put sensor and host state there. That is the V-374 edit
|
||||
of 31-07-2026, which the seeds never received.
|
||||
- **world questions, 8 lines.** `почему небо голубое`, `why is the sky blue`. Written when
|
||||
chat was the only honest destination for a question nothing could answer, and external
|
||||
search now answers them.
|
||||
- **bare verbs, 8 lines.** `поставь напоминание` with nothing to remind about. The prompt
|
||||
calls that unknown. This one is not staleness. A nearest-neighbour centroid wants the
|
||||
bare verb phrase, and that is what a seed file is for.
|
||||
|
||||
Four intents have not been redefined since the seeds were written: note, fact, query and
|
||||
act. They agree at 178 of 191.
|
||||
|
||||
## What this says about the plan
|
||||
|
||||
Gemma is usable as a label function on those four and not on system, chat or a bare verb.
|
||||
The plan already budgets a day of the owner reading the set. This says where to spend it.
|
||||
|
||||
It also says the two engines in the cascade are being taught different rules on 80 lines.
|
||||
A routing measurement that swaps between the classifier and the router is measuring some of
|
||||
that disagreement rather than the models.
|
||||
@@ -0,0 +1,58 @@
|
||||
# Moving the seed files onto the router prompt's boundaries
|
||||
|
||||
**06-08-2026. V-626.** Measured with `TestONNXBaseline`, 91-case RU routing fixture,
|
||||
classifier plus the ONNX embedder. No LLM arm in this run.
|
||||
|
||||
`docs/evals/2026-08-06-seed-labels-vs-router-prompt.md` found three intent boundaries where
|
||||
`models/seeds` and `routeSystem` disagree. This applies two of them and rejects the third,
|
||||
because the third was measured and it costs a case.
|
||||
|
||||
## Baseline
|
||||
|
||||
**64/91, 70.3% full.** Latency p50 22.9ms.
|
||||
|
||||
## What moved
|
||||
|
||||
**Sensor and host state, system to query. 26 lines.** `какая температура воздуха`,
|
||||
`сколько памяти занято`, `какой статус сервисов`. The prompt restricts system to the clock,
|
||||
the calendar date and the assistant itself, which is the V-374 edit of 31-07-2026.
|
||||
|
||||
**World questions, chat to query. 8 lines.** `почему небо голубое`, `why is the sky blue`,
|
||||
`как работает интернет`. Only the genuine world-knowledge lines. An opener about herself
|
||||
stays in chat. `как тебя зовут` is a question word by rule 4 and about the assistant by
|
||||
rule 8. The rules are ordered and rule 4 fires first, which reads wrong. That is a prompt
|
||||
question rather than a seed question.
|
||||
|
||||
`system.txt` goes from 43 lines to 17 and `query.txt` from 64 to 98.
|
||||
|
||||
## Result
|
||||
|
||||
**66/91, 72.5% full.** Two cases gained, none lost.
|
||||
|
||||
- `en-sys-002` "turn quiet mode back on", quiet 2/3 to 3/3
|
||||
- `ru-query-011` "почему сервер тормозит", homelab 5/6 to 6/6
|
||||
|
||||
Clarify counts unchanged at 0 false and 8 missed. The eight missed clarifies are the
|
||||
`ambiguous` tag and this change does not touch them. `TestONNXRecall`, `TestONNXTopics`,
|
||||
`TestONNXPersonalBoundary` and `TestONNXClaimConfidenceDistribution` all pass.
|
||||
|
||||
Thinning system to 17 lines did not hurt it. The two remaining system failures,
|
||||
`какое число завтра` and `какой день недели послезавтра`, both failed at baseline too.
|
||||
|
||||
## The third boundary, measured and rejected
|
||||
|
||||
`reminder.txt` holds eight bare verbs: `поставь напоминание`, `создай напоминание`,
|
||||
`set a reminder`. Rule 9 of the prompt calls an utterance with no named subject unknown.
|
||||
By the prompt they do not belong in a reminder seed set.
|
||||
|
||||
Dropping them scores **65/91**, one below keeping them. `ru-rem-004` "поставь напоминание
|
||||
через полчаса" falls from reminder to fact, because the centroid loses the phrase the
|
||||
utterance is built from.
|
||||
|
||||
So the seed file and the prompt are not stale against each other here. They have different
|
||||
jobs. A prompt classifies one utterance and can say it cannot. A nearest-neighbour centroid
|
||||
is a shape to be near, and a bare verb phrase is part of that shape. The eight lines stay.
|
||||
|
||||
That distinction matters past this file. V-546 trains a classification head on labeled
|
||||
utterances rather than a centroid, and the head is the prompt's kind of thing. These eight
|
||||
lines are seed data and not training data.
|
||||
@@ -173,10 +173,11 @@
|
||||
]
|
||||
},
|
||||
"reminder_verbs": {
|
||||
"note": "The imperatives that mean \"remind me\", in the forms he speaks. The same kind of set as capture_verbs and decided the same way: it is her vocabulary, not a discovery about Russian (Vikunja #530).",
|
||||
"note": "The imperatives that mean \"remind me\", in the forms he speaks. The same kind of set as capture_verbs and decided the same way: it is her vocabulary, not a discovery about Russian (Vikunja #530). The alarm verbs joined them in V-627. \"разбуди меня в 6:30\" is a reminder that fires at the hour he gets up, and the set knew no form of it, so an alarm reached IntentReminder only by resembling one to the embedder.",
|
||||
"words": [
|
||||
"напомни", "напомните", "напомнить", "напоминай",
|
||||
"remind"
|
||||
"разбуди", "разбудите", "разбудить", "буди",
|
||||
"remind", "wake"
|
||||
]
|
||||
},
|
||||
"half_hour": {
|
||||
|
||||
@@ -662,7 +662,7 @@ func (t *emptyFrameTransport) Call(ctx context.Context, req *rpcRequest) (*rpcRe
|
||||
}
|
||||
|
||||
func (t *emptyFrameTransport) Notify(context.Context, string, any) error { return nil }
|
||||
func (t *emptyFrameTransport) Close() error { return nil }
|
||||
func (t *emptyFrameTransport) Close() error { return nil }
|
||||
|
||||
func TestResultlessResponseIsNotSuccess(t *testing.T) {
|
||||
c := newClient("empty", &emptyFrameTransport{})
|
||||
|
||||
@@ -2,6 +2,7 @@ package router
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
@@ -94,10 +95,41 @@ func DefaultGrammars(actMatcher ActMatcher) []Grammar {
|
||||
// stage-0 decision too (fillMatchedSlots in router.go). Before that it did not,
|
||||
// so "напомни в 11:00 позвонить маме" reached the daemon with HasTime false and
|
||||
// was asked "Когда?" about an hour he had just said.
|
||||
//
|
||||
// The verb alternation is built from lexicon.ReminderVerbs rather than written
|
||||
// out (V-627). The literal here knew "напомни" and "remind me" and nothing
|
||||
// else, so "разбуди меня в 6:30" never reached stage 0 — and it does not reach
|
||||
// IntentReminder further down either, where the classifier calls it fact at
|
||||
// 0.918. An alarm is a reminder that fires at the hour he gets up, and the
|
||||
// verb that names one is her vocabulary, so it belongs in the lexicon with the
|
||||
// rest of it.
|
||||
//
|
||||
// Longest-first ordering matters: Go's regexp alternation is leftmost-first,
|
||||
// not longest-match, so "напомнить" listed after "напомни" would never match.
|
||||
var reminderVerbPattern = regexp.MustCompile(
|
||||
`(?i)^\s*(?:` + longestFirstAlternation(lexicon.ReminderVerbs()) +
|
||||
`)\s*(?:мне|меня|me)?[\s,:]+(.+)$`)
|
||||
|
||||
// longestFirstAlternation joins a word set into a regexp alternation, longest
|
||||
// alternative first, with every member escaped.
|
||||
func longestFirstAlternation(set []string) string {
|
||||
out := make([]string, 0, len(set))
|
||||
for _, w := range set {
|
||||
out = append(out, regexp.QuoteMeta(w))
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if len(out[i]) != len(out[j]) {
|
||||
return len(out[i]) > len(out[j])
|
||||
}
|
||||
return out[i] < out[j]
|
||||
})
|
||||
return strings.Join(out, "|")
|
||||
}
|
||||
|
||||
func ReminderGrammar() Grammar {
|
||||
return Grammar{
|
||||
Name: "reminder-wakeword",
|
||||
Pattern: regexp.MustCompile(`(?i)^\s*(?:напомни|remind me)[\s,:]+(.+)$`),
|
||||
Pattern: reminderVerbPattern,
|
||||
Build: func(m []string) (Decision, bool) {
|
||||
rest := strings.TrimSpace(m[1])
|
||||
if rest == "" {
|
||||
|
||||
@@ -8,12 +8,9 @@
|
||||
как прошёл день
|
||||
расскажи про себя
|
||||
ты мне нравишься
|
||||
почему небо голубое
|
||||
о чём поговорим
|
||||
у тебя есть чувства
|
||||
что такое любовь
|
||||
расскажи историю
|
||||
как работает интернет
|
||||
шутка
|
||||
анекдот
|
||||
пошути
|
||||
@@ -24,8 +21,6 @@
|
||||
что нового
|
||||
думаешь о чём-то
|
||||
расскажи про космос
|
||||
почему трава зелёная
|
||||
откуда берётся дождь
|
||||
что было интересного сегодня
|
||||
как тебя зовут
|
||||
сколько тебе лет
|
||||
@@ -37,7 +32,4 @@ i'm bored
|
||||
what's up
|
||||
tell me a joke
|
||||
do you have feelings
|
||||
what is love
|
||||
tell me about yourself
|
||||
why is the sky blue
|
||||
how does the internet work
|
||||
|
||||
@@ -62,3 +62,37 @@ what did I note about the garden
|
||||
будет дождь
|
||||
погода на сегодня
|
||||
weather in london
|
||||
сколько времени осталось до вечера
|
||||
какая температура воздуха
|
||||
сколько человек дома
|
||||
кто сейчас дома
|
||||
есть ли кто дома
|
||||
кто дома сейчас
|
||||
все ли дома
|
||||
сколько памяти занято
|
||||
какая загрузка процессора
|
||||
сколько свободного места на диске
|
||||
какой ip адрес у сервера
|
||||
всё ли работает
|
||||
сколько сервер работает без перезагрузки
|
||||
когда сервер запускался
|
||||
какая версия софта
|
||||
сколько аптайм
|
||||
какой статус сервисов
|
||||
все ли сервисы работают
|
||||
что с интернетом
|
||||
когда последний раз перезагружался
|
||||
сколько трафика сегодня
|
||||
какая скорость интернета
|
||||
сколько процессов запущено
|
||||
как загрузка системы
|
||||
сколько оперативной памяти свободно
|
||||
какая температура процессора
|
||||
почему небо голубое
|
||||
что такое любовь
|
||||
как работает интернет
|
||||
почему трава зелёная
|
||||
откуда берётся дождь
|
||||
what is love
|
||||
why is the sky blue
|
||||
how does the internet work
|
||||
|
||||
+1
-27
@@ -5,36 +5,10 @@
|
||||
который час в Москве
|
||||
сколько сейчас времени
|
||||
который час у нас
|
||||
сколько времени осталось до вечера
|
||||
какой сегодня день недели
|
||||
какая температура воздуха
|
||||
сколько человек дома
|
||||
кто сейчас дома
|
||||
есть ли кто дома
|
||||
кто дома сейчас
|
||||
все ли дома
|
||||
сколько памяти занято
|
||||
какая загрузка процессора
|
||||
сколько свободного места на диске
|
||||
какой ip адрес у сервера
|
||||
как дела у сервера
|
||||
всё ли работает
|
||||
сколько сервер работает без перезагрузки
|
||||
когда сервер запускался
|
||||
какая версия софта
|
||||
сколько аптайм
|
||||
какой статус сервисов
|
||||
все ли сервисы работают
|
||||
что с интернетом
|
||||
интернет работает
|
||||
когда последний раз перезагружался
|
||||
сколько трафика сегодня
|
||||
какая скорость интернета
|
||||
загрузка сети
|
||||
сколько процессов запущено
|
||||
как загрузка системы
|
||||
сколько оперативной памяти свободно
|
||||
какая температура процессора
|
||||
тихий режим
|
||||
тихо
|
||||
не шуми
|
||||
@@ -48,4 +22,4 @@
|
||||
quiet mode on
|
||||
quiet mode off
|
||||
quiet on
|
||||
quiet off
|
||||
quiet off
|
||||
|
||||
Reference in New Issue
Block a user