Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2b3e34c7e8 |
@@ -0,0 +1,113 @@
|
|||||||
|
// Command labelgen labels utterances with the stage 0 grammars and prints JSONL.
|
||||||
|
//
|
||||||
|
// docs/plans/18-routing-heads-on-e5-small.md calls the labeled set the whole
|
||||||
|
// project, and it names the stage 0 grammars as the high-precision label
|
||||||
|
// functions to start from. This runs them — the real ones, in the real
|
||||||
|
// buildRouter order — rather than a reimplementation, so a rule change moves
|
||||||
|
// the training data with it.
|
||||||
|
//
|
||||||
|
// A grammar that declines leaves the line unlabeled. Those go to the model, and
|
||||||
|
// keeping them is the point: a set labeled only by the rules teaches only the
|
||||||
|
// rules.
|
||||||
|
//
|
||||||
|
// go run ./cmd/labelgen < utterances.txt > labeled.jsonl
|
||||||
|
//
|
||||||
|
// The wakeword-act grammar is absent, because its allowlist is the deployment's
|
||||||
|
// enabled tool names and this tool has no deployment. Every other rule is here.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/kami/maven/internal/router"
|
||||||
|
)
|
||||||
|
|
||||||
|
// label is one output row. The grammar name rides along so a reviewer can see
|
||||||
|
// which rule made the claim, and so a rule that turns out to be wrong can have
|
||||||
|
// its rows pulled without re-running everything.
|
||||||
|
type label struct {
|
||||||
|
Utterance string `json:"utterance"`
|
||||||
|
Intent string `json:"intent,omitempty"`
|
||||||
|
Grammar string `json:"grammar,omitempty"`
|
||||||
|
Key string `json:"key,omitempty"`
|
||||||
|
Value string `json:"value,omitempty"`
|
||||||
|
Fn string `json:"fn,omitempty"`
|
||||||
|
Text string `json:"text,omitempty"`
|
||||||
|
Labeled bool `json:"labeled"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// grammars mirrors buildRouter's order in cmd/mavend/voicewire.go. Order is
|
||||||
|
// load-bearing there and so it is here: the agenda rules must sit after the
|
||||||
|
// clock rules, Praxis before the capture marker, the narrative rules last.
|
||||||
|
func grammars() []router.Grammar {
|
||||||
|
var g []router.Grammar
|
||||||
|
g = append(g, router.SystemTimeDateGrammars()...)
|
||||||
|
g = append(g, router.AgendaQueryGrammars()...)
|
||||||
|
g = append(g, router.FeedQueryGrammar())
|
||||||
|
g = append(g, router.TaskListGrammar())
|
||||||
|
g = append(g, router.ListGrammars()...)
|
||||||
|
g = append(g, router.ReminderGrammar())
|
||||||
|
g = append(g, router.PraxisGrammars()...)
|
||||||
|
g = append(g, router.TaskCaptureGrammar())
|
||||||
|
g = append(g, router.NarrativeQueryGrammars()...)
|
||||||
|
return g
|
||||||
|
}
|
||||||
|
|
||||||
|
func match(gs []router.Grammar, utterance string) label {
|
||||||
|
out := label{Utterance: utterance}
|
||||||
|
for _, g := range gs {
|
||||||
|
m := g.Pattern.FindStringSubmatch(utterance)
|
||||||
|
if m == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
d, ok := g.Build(m)
|
||||||
|
if !ok {
|
||||||
|
continue // the rule saw its shape and declined it
|
||||||
|
}
|
||||||
|
out.Intent = string(d.Intent)
|
||||||
|
out.Grammar = g.Name
|
||||||
|
out.Key = d.Slots.Key
|
||||||
|
out.Value = d.Slots.Value
|
||||||
|
out.Fn = d.Slots.Fn
|
||||||
|
out.Text = d.Slots.Text
|
||||||
|
out.Labeled = true
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
gs := grammars()
|
||||||
|
in := bufio.NewScanner(os.Stdin)
|
||||||
|
in.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||||
|
out := bufio.NewWriter(os.Stdout)
|
||||||
|
defer out.Flush()
|
||||||
|
|
||||||
|
enc := json.NewEncoder(out)
|
||||||
|
var seen, labeled int
|
||||||
|
for in.Scan() {
|
||||||
|
line := strings.TrimSpace(in.Text())
|
||||||
|
if line == "" || strings.HasPrefix(line, "#") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen++
|
||||||
|
l := match(gs, line)
|
||||||
|
if l.Labeled {
|
||||||
|
labeled++
|
||||||
|
}
|
||||||
|
if err := enc.Encode(l); err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, "labelgen:", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := in.Err(); err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, "labelgen:", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
// Coverage on stderr, so the count is visible without polluting the JSONL.
|
||||||
|
fmt.Fprintf(os.Stderr, "labelgen: %d/%d labeled by %d grammars\n", labeled, seen, len(gs))
|
||||||
|
}
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
# Gemma as a label function, and what it found in the seeds
|
||||||
|
|
||||||
|
**06-08-2026. V-546.** Measured on workpc against gemma-4-12b-it-qat-UD-Q4_K_XL.
|
||||||
|
|
||||||
|
`docs/plans/18-routing-heads-on-e5-small.md` puts the labeled set at 20k examples through
|
||||||
|
gemma, costing 2 to 4 hours of the card. This is the check before spending that. Gemma
|
||||||
|
labels the 344 hand-written classifier seeds. Agreement with the label a person already
|
||||||
|
chose is a precision number rather than a guess.
|
||||||
|
|
||||||
|
## What ran
|
||||||
|
|
||||||
|
`cmd/labelgen` runs the stage 0 grammars. The real ones, in `buildRouter` order, minus
|
||||||
|
`wakeword-act`, whose allowlist is a deployment's enabled tool names. It labels 62 of 339
|
||||||
|
seed lines and leaves the rest.
|
||||||
|
|
||||||
|
The remaining 277 went to gemma through the daemon's own `routeSystem` prompt and
|
||||||
|
`routeGrammar`, both extracted from `internal/router/llmrouter.go` at run time rather than
|
||||||
|
retyped. Temperature 0.
|
||||||
|
|
||||||
|
## Cost
|
||||||
|
|
||||||
|
**334ms per call, 0 unparsed of 277.** The GBNF held every time. At that rate the plan's
|
||||||
|
20k examples is under two hours of card, which matches its estimate.
|
||||||
|
|
||||||
|
## The stage 0 rules as label functions
|
||||||
|
|
||||||
|
Agreement between the grammar's label and the seed file the line came from:
|
||||||
|
|
||||||
|
| seed intent | agree |
|
||||||
|
|---|---|
|
||||||
|
| reminder | 37/37 |
|
||||||
|
| query | 9/10 |
|
||||||
|
| system | 7/8 |
|
||||||
|
| act | 2/2 |
|
||||||
|
| chat | 0/4 |
|
||||||
|
| note | 0/1 |
|
||||||
|
|
||||||
|
`ReminderGrammar` at 37/37 is the evidence the plan wanted. The chat column is a defect
|
||||||
|
rather than a disagreement: `chatNarrativeTopics` is Russian-only, so `tell me about
|
||||||
|
yourself` survives the decline and routes IntentQuery with topic `yourself`. Filed as
|
||||||
|
V-625, which also records that `как дела у сервера` appears verbatim in two seed files
|
||||||
|
under two intents.
|
||||||
|
|
||||||
|
## Gemma against the seeds
|
||||||
|
|
||||||
|
**197/277, 71.1%.** By intent:
|
||||||
|
|
||||||
|
| seed intent | agree |
|
||||||
|
|---|---|
|
||||||
|
| note | 33/33 |
|
||||||
|
| act | 57/64 |
|
||||||
|
| fact | 37/40 |
|
||||||
|
| query | 51/54 |
|
||||||
|
| chat | 15/35 |
|
||||||
|
| system | 4/43 |
|
||||||
|
| reminder | 0/8 |
|
||||||
|
|
||||||
|
The number is not gemma's error rate. Reading the 80 disagreements, most are the seed files
|
||||||
|
and the prompt holding different definitions of the same intent. Three boundaries carry 42
|
||||||
|
of them, and V-626 is the fix:
|
||||||
|
|
||||||
|
- **system, 26 lines.** The prompt restricts system to the clock, the calendar date and the
|
||||||
|
assistant itself. The seeds also put sensor and host state there. That is the V-374 edit
|
||||||
|
of 31-07-2026, which the seeds never received.
|
||||||
|
- **world questions, 8 lines.** `почему небо голубое`, `why is the sky blue`. Written when
|
||||||
|
chat was the only honest destination for a question nothing could answer, and external
|
||||||
|
search now answers them.
|
||||||
|
- **bare verbs, 8 lines.** `поставь напоминание` with nothing to remind about. The prompt
|
||||||
|
calls that unknown. This one is not staleness. A nearest-neighbour centroid wants the
|
||||||
|
bare verb phrase, and that is what a seed file is for.
|
||||||
|
|
||||||
|
Four intents have not been redefined since the seeds were written: note, fact, query and
|
||||||
|
act. They agree at 178 of 191.
|
||||||
|
|
||||||
|
## What this says about the plan
|
||||||
|
|
||||||
|
Gemma is usable as a label function on those four and not on system, chat or a bare verb.
|
||||||
|
The plan already budgets a day of the owner reading the set. This says where to spend it.
|
||||||
|
|
||||||
|
It also says the two engines in the cascade are being taught different rules on 80 lines.
|
||||||
|
A routing measurement that swaps between the classifier and the router is measuring some of
|
||||||
|
that disagreement rather than the models.
|
||||||
Reference in New Issue
Block a user