From 2b3e34c7e8d8c50104e120bd404b14a450fcb7f1 Mon Sep 17 00:00:00 2001 From: claude Date: Thu, 6 Aug 2026 13:21:06 +0400 Subject: [PATCH] label seeds with the stage 0 grammars and gemma, and measure both (V-546) The plan calls the labeled set the whole project and names the stage 0 grammars as the label functions. cmd/labelgen runs them, the real ones in buildRouter order, so a rule change moves the training data with it. Gemma labels the rest at 334ms/call with nothing unparsed, which matches the plan's estimate. It agrees with the seed files on 197/277, and reading the disagreements is the finding: the seeds and the router prompt hold different definitions of system, of a world question and of a bare verb. V-626. --- cmd/labelgen/main.go | 113 ++++++++++++++++++ ...2026-08-06-seed-labels-vs-router-prompt.md | 82 +++++++++++++ 2 files changed, 195 insertions(+) create mode 100644 cmd/labelgen/main.go create mode 100644 docs/evals/2026-08-06-seed-labels-vs-router-prompt.md diff --git a/cmd/labelgen/main.go b/cmd/labelgen/main.go new file mode 100644 index 0000000..801013a --- /dev/null +++ b/cmd/labelgen/main.go @@ -0,0 +1,113 @@ +// Command labelgen labels utterances with the stage 0 grammars and prints JSONL. +// +// docs/plans/18-routing-heads-on-e5-small.md calls the labeled set the whole +// project, and it names the stage 0 grammars as the high-precision label +// functions to start from. This runs them — the real ones, in the real +// buildRouter order — rather than a reimplementation, so a rule change moves +// the training data with it. +// +// A grammar that declines leaves the line unlabeled. Those go to the model, and +// keeping them is the point: a set labeled only by the rules teaches only the +// rules. +// +// go run ./cmd/labelgen < utterances.txt > labeled.jsonl +// +// The wakeword-act grammar is absent, because its allowlist is the deployment's +// enabled tool names and this tool has no deployment. Every other rule is here. +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/kami/maven/internal/router" +) + +// label is one output row. The grammar name rides along so a reviewer can see +// which rule made the claim, and so a rule that turns out to be wrong can have +// its rows pulled without re-running everything. +type label struct { + Utterance string `json:"utterance"` + Intent string `json:"intent,omitempty"` + Grammar string `json:"grammar,omitempty"` + Key string `json:"key,omitempty"` + Value string `json:"value,omitempty"` + Fn string `json:"fn,omitempty"` + Text string `json:"text,omitempty"` + Labeled bool `json:"labeled"` +} + +// grammars mirrors buildRouter's order in cmd/mavend/voicewire.go. Order is +// load-bearing there and so it is here: the agenda rules must sit after the +// clock rules, Praxis before the capture marker, the narrative rules last. +func grammars() []router.Grammar { + var g []router.Grammar + g = append(g, router.SystemTimeDateGrammars()...) + g = append(g, router.AgendaQueryGrammars()...) + g = append(g, router.FeedQueryGrammar()) + g = append(g, router.TaskListGrammar()) + g = append(g, router.ListGrammars()...) + g = append(g, router.ReminderGrammar()) + g = append(g, router.PraxisGrammars()...) + g = append(g, router.TaskCaptureGrammar()) + g = append(g, router.NarrativeQueryGrammars()...) + return g +} + +func match(gs []router.Grammar, utterance string) label { + out := label{Utterance: utterance} + for _, g := range gs { + m := g.Pattern.FindStringSubmatch(utterance) + if m == nil { + continue + } + d, ok := g.Build(m) + if !ok { + continue // the rule saw its shape and declined it + } + out.Intent = string(d.Intent) + out.Grammar = g.Name + out.Key = d.Slots.Key + out.Value = d.Slots.Value + out.Fn = d.Slots.Fn + out.Text = d.Slots.Text + out.Labeled = true + return out + } + return out +} + +func main() { + gs := grammars() + in := bufio.NewScanner(os.Stdin) + in.Buffer(make([]byte, 0, 64*1024), 1024*1024) + out := bufio.NewWriter(os.Stdout) + defer out.Flush() + + enc := json.NewEncoder(out) + var seen, labeled int + for in.Scan() { + line := strings.TrimSpace(in.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + seen++ + l := match(gs, line) + if l.Labeled { + labeled++ + } + if err := enc.Encode(l); err != nil { + fmt.Fprintln(os.Stderr, "labelgen:", err) + os.Exit(1) + } + } + if err := in.Err(); err != nil { + fmt.Fprintln(os.Stderr, "labelgen:", err) + os.Exit(1) + } + // Coverage on stderr, so the count is visible without polluting the JSONL. + fmt.Fprintf(os.Stderr, "labelgen: %d/%d labeled by %d grammars\n", labeled, seen, len(gs)) +} diff --git a/docs/evals/2026-08-06-seed-labels-vs-router-prompt.md b/docs/evals/2026-08-06-seed-labels-vs-router-prompt.md new file mode 100644 index 0000000..e501e11 --- /dev/null +++ b/docs/evals/2026-08-06-seed-labels-vs-router-prompt.md @@ -0,0 +1,82 @@ +# Gemma as a label function, and what it found in the seeds + +**06-08-2026. V-546.** Measured on workpc against gemma-4-12b-it-qat-UD-Q4_K_XL. + +`docs/plans/18-routing-heads-on-e5-small.md` puts the labeled set at 20k examples through +gemma, costing 2 to 4 hours of the card. This is the check before spending that. Gemma +labels the 344 hand-written classifier seeds. Agreement with the label a person already +chose is a precision number rather than a guess. + +## What ran + +`cmd/labelgen` runs the stage 0 grammars. The real ones, in `buildRouter` order, minus +`wakeword-act`, whose allowlist is a deployment's enabled tool names. It labels 62 of 339 +seed lines and leaves the rest. + +The remaining 277 went to gemma through the daemon's own `routeSystem` prompt and +`routeGrammar`, both extracted from `internal/router/llmrouter.go` at run time rather than +retyped. Temperature 0. + +## Cost + +**334ms per call, 0 unparsed of 277.** The GBNF held every time. At that rate the plan's +20k examples is under two hours of card, which matches its estimate. + +## The stage 0 rules as label functions + +Agreement between the grammar's label and the seed file the line came from: + +| seed intent | agree | +|---|---| +| reminder | 37/37 | +| query | 9/10 | +| system | 7/8 | +| act | 2/2 | +| chat | 0/4 | +| note | 0/1 | + +`ReminderGrammar` at 37/37 is the evidence the plan wanted. The chat column is a defect +rather than a disagreement: `chatNarrativeTopics` is Russian-only, so `tell me about +yourself` survives the decline and routes IntentQuery with topic `yourself`. Filed as +V-625, which also records that `как дела у сервера` appears verbatim in two seed files +under two intents. + +## Gemma against the seeds + +**197/277, 71.1%.** By intent: + +| seed intent | agree | +|---|---| +| note | 33/33 | +| act | 57/64 | +| fact | 37/40 | +| query | 51/54 | +| chat | 15/35 | +| system | 4/43 | +| reminder | 0/8 | + +The number is not gemma's error rate. Reading the 80 disagreements, most are the seed files +and the prompt holding different definitions of the same intent. Three boundaries carry 42 +of them, and V-626 is the fix: + +- **system, 26 lines.** The prompt restricts system to the clock, the calendar date and the + assistant itself. The seeds also put sensor and host state there. That is the V-374 edit + of 31-07-2026, which the seeds never received. +- **world questions, 8 lines.** `почему небо голубое`, `why is the sky blue`. Written when + chat was the only honest destination for a question nothing could answer, and external + search now answers them. +- **bare verbs, 8 lines.** `поставь напоминание` with nothing to remind about. The prompt + calls that unknown. This one is not staleness. A nearest-neighbour centroid wants the + bare verb phrase, and that is what a seed file is for. + +Four intents have not been redefined since the seeds were written: note, fact, query and +act. They agree at 178 of 191. + +## What this says about the plan + +Gemma is usable as a label function on those four and not on system, chat or a bare verb. +The plan already budgets a day of the owner reading the set. This says where to spend it. + +It also says the two engines in the cascade are being taught different rules on 80 lines. +A routing measurement that swaps between the classifier and the router is measuring some of +that disagreement rather than the models.