// Command labelgen labels utterances with the stage 0 grammars and prints JSONL. // // docs/plans/18-routing-heads-on-e5-small.md calls the labeled set the whole // project, and it names the stage 0 grammars as the high-precision label // functions to start from. This runs them — the real ones, in the real // buildRouter order — rather than a reimplementation, so a rule change moves // the training data with it. // // A grammar that declines leaves the line unlabeled. Those go to the model, and // keeping them is the point: a set labeled only by the rules teaches only the // rules. // // go run ./cmd/labelgen < utterances.txt > labeled.jsonl // // The wakeword-act grammar is absent, because its allowlist is the deployment's // enabled tool names and this tool has no deployment. Every other rule is here. package main import ( "bufio" "encoding/json" "fmt" "os" "strings" "github.com/kami/maven/internal/router" ) // label is one output row. The grammar name rides along so a reviewer can see // which rule made the claim, and so a rule that turns out to be wrong can have // its rows pulled without re-running everything. type label struct { Utterance string `json:"utterance"` Intent string `json:"intent,omitempty"` Grammar string `json:"grammar,omitempty"` Key string `json:"key,omitempty"` Value string `json:"value,omitempty"` Fn string `json:"fn,omitempty"` Text string `json:"text,omitempty"` Labeled bool `json:"labeled"` } // grammars is the daemon's canonical ordered stage-zero set. Label generation // must not maintain a second copy: that drift was the defect fixed by V-693. func grammars() []router.Grammar { return router.StageZeroGrammars(router.DefaultActMatcher{}) } func match(gs []router.Grammar, utterance string) label { out := label{Utterance: utterance} for _, g := range gs { d, matched, ok := g.Evaluate(utterance) if !matched || !ok { continue } out.Intent = string(d.Intent) out.Grammar = g.Name out.Key = d.Slots.Key out.Value = d.Slots.Value out.Fn = d.Slots.Fn out.Text = d.Slots.Text out.Labeled = true return out } return out } func main() { gs := grammars() in := bufio.NewScanner(os.Stdin) in.Buffer(make([]byte, 0, 64*1024), 1024*1024) out := bufio.NewWriter(os.Stdout) defer out.Flush() enc := json.NewEncoder(out) var seen, labeled int for in.Scan() { line := strings.TrimSpace(in.Text()) if line == "" || strings.HasPrefix(line, "#") { continue } seen++ l := match(gs, line) if l.Labeled { labeled++ } if err := enc.Encode(l); err != nil { fmt.Fprintln(os.Stderr, "labelgen:", err) os.Exit(1) } } if err := in.Err(); err != nil { fmt.Fprintln(os.Stderr, "labelgen:", err) os.Exit(1) } // Coverage on stderr, so the count is visible without polluting the JSONL. fmt.Fprintf(os.Stderr, "labelgen: %d/%d labeled by %d grammars\n", labeled, seen, len(gs)) }