router/semantic: slice 22 residual non-action router — emit step, Go harness (legacy + heads modes), Python experiment
This commit is contained in:
@@ -0,0 +1,64 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/kami/maven/internal/router"
|
||||
)
|
||||
|
||||
// headsMain runs the deployed cascade minus the resident LLM: stage-0
|
||||
// grammars → routing heads (fine-tuned e5 copy + softmax, router_heads.onnx,
|
||||
// 0.6 decline threshold) → ONNX-embedder nearest-centroid classifier →
|
||||
// 0.55 confidence gate. This is what a production turn takes when the model
|
||||
// server is out (docs/routing.md: pickLLMRouter degrades to the classifier).
|
||||
//
|
||||
// The classifier is seeded from models/seeds like the daemon's seedClassifier,
|
||||
// embedded with the real multilingual-e5-small model rather than the block
|
||||
// hash, so this is the closest headless reproduction of the authoritative
|
||||
// router output the slice-22 report can run.
|
||||
//
|
||||
// Requires the ONNX model files and a libonnxruntime.so. Pass the library via
|
||||
// the MAVEN_ONNX_LIB environment variable, exactly as the daemon does.
|
||||
func headsMain(poolPath, outPath string) {
|
||||
lib := os.Getenv("MAVEN_ONNX_LIB")
|
||||
if lib == "" {
|
||||
fmt.Fprintln(os.Stderr, "heads mode needs MAVEN_ONNX_LIB pointing at libonnxruntime.so")
|
||||
os.Exit(2)
|
||||
}
|
||||
const (
|
||||
embedModel = "models/embedder/multilingual-e5-small/model_quantized.onnx"
|
||||
tokPath = "models/embedder/multilingual-e5-small/tokenizer.json"
|
||||
headsModel = "models/embedder/router-heads/router_heads.onnx"
|
||||
)
|
||||
emb, err := router.NewONNXEmbedder(embedModel, tokPath, lib)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "heads: embedder: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer emb.Close()
|
||||
|
||||
cls := router.NewClassifier(emb)
|
||||
seedClassifier(cls)
|
||||
|
||||
heads, err := router.NewRouterHeads(headsModel, tokPath)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "heads: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer heads.Close()
|
||||
|
||||
acts := router.DefaultActMatcher{Fns: actVerbList()}
|
||||
r := router.New(router.Config{
|
||||
Grammars: router.StageZeroGrammars(acts),
|
||||
Classifier: cls,
|
||||
Extractor: router.Extractor{
|
||||
Time: router.StubDateTimeParser{},
|
||||
Acts: acts,
|
||||
Facts: router.DefaultFactParser{},
|
||||
},
|
||||
Threshold: 0.55,
|
||||
Heads: heads,
|
||||
})
|
||||
runOverPool(r, poolPath, outPath)
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/kami/maven/internal/router"
|
||||
)
|
||||
|
||||
// Seed loading replicated from cmd/mavend/voicewire.go (seedClassifier) and
|
||||
// internal/router/semantic/helpers_test.go, which build the same classifier
|
||||
// from models/seeds/<intent>.txt. The daemon and the eval fixture must agree
|
||||
// on the seeds; so must a measurement.
|
||||
const seedDir = "models/seeds"
|
||||
|
||||
var seedIntents = []router.Intent{
|
||||
router.IntentAct, router.IntentReminder, router.IntentFact,
|
||||
router.IntentNote, router.IntentQuery, router.IntentChat, router.IntentSystem,
|
||||
}
|
||||
|
||||
// actVerbList mirrors the eval fixture's static allowlist
|
||||
// (internal/router/semantic/helpers_test.go). The production matcher's
|
||||
// allowlist is the deployment's enabled tools; an act grammar can only catch
|
||||
// a row whose first tokens match an allowlisted verb, and the experiment's
|
||||
// act allowlist is the one the accepted slice-21 methodology used.
|
||||
func actVerbList() []string {
|
||||
return []string{
|
||||
"перезапусти", "перезагрузи", "выключи", "включи", "останови", "запусти",
|
||||
"закрой", "открой", "сделай", "поставь",
|
||||
"restart", "reboot", "stop", "start", "turn off", "turn on", "open", "close",
|
||||
}
|
||||
}
|
||||
|
||||
// buildMinimalRouter reproduces internal/router/semantic/buildMinimalRouter:
|
||||
// the daemon's grammar set, a hash-embedder classifier seeded from
|
||||
// models/seeds, and the deployed 0.55 threshold. Deterministic and
|
||||
// reproducible. The ONNX embedder and the routing heads score elsewhere;
|
||||
// this is the floor the eval fixture reports as the legacy baseline.
|
||||
func buildMinimalRouter() *router.Router {
|
||||
acts := router.DefaultActMatcher{Fns: actVerbList()}
|
||||
cls := router.NewClassifier(router.NewHashEmbedder(1024))
|
||||
seedClassifier(cls)
|
||||
return router.New(router.Config{
|
||||
Grammars: router.StageZeroGrammars(acts),
|
||||
Classifier: cls,
|
||||
Extractor: router.Extractor{
|
||||
Time: router.StubDateTimeParser{},
|
||||
Acts: acts,
|
||||
Facts: router.DefaultFactParser{},
|
||||
},
|
||||
Threshold: 0.55,
|
||||
})
|
||||
}
|
||||
|
||||
func seedClassifier(c *router.Classifier) {
|
||||
// Walk up to find models/seeds like the daemon's seedPath, so the program
|
||||
// can run from any depth of the repo tree.
|
||||
dir := seedDir
|
||||
for i := 0; i < 5; i++ {
|
||||
if st, err := os.Stat(dir); err == nil && st.IsDir() {
|
||||
break
|
||||
}
|
||||
dir = filepath.Join("..", dir)
|
||||
}
|
||||
ctx := context.Background()
|
||||
total := 0
|
||||
for _, intent := range seedIntents {
|
||||
path := filepath.Join(dir, string(intent)+".txt")
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
log.Printf("legacy: open seed %s: %v", path, err)
|
||||
continue
|
||||
}
|
||||
sc := bufio.NewScanner(f)
|
||||
lines := []string{}
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
lines = append(lines, line)
|
||||
}
|
||||
f.Close()
|
||||
sort.Strings(lines)
|
||||
for _, line := range lines {
|
||||
if err := c.AddExample(ctx, intent, line); err != nil {
|
||||
log.Printf("legacy: seed %s %q: %v", intent, line, err)
|
||||
continue
|
||||
}
|
||||
total++
|
||||
}
|
||||
}
|
||||
log.Printf("legacy: loaded %d seed examples from %s", total, dir)
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/router"
|
||||
)
|
||||
|
||||
// Legacy-baseline runner for slice 22: run the actual router cascade (stage 0
|
||||
// grammars → hash-embedder nearest-centroid classifier → 0.55 confidence gate)
|
||||
// over the frozen residual non-action dev pool and project each decision into
|
||||
// the five-way non-action semantic space.
|
||||
//
|
||||
// Projection rules (the daemon's behaviour, not just ScoreLegacy's):
|
||||
// - route error → uncertain
|
||||
// - Clarify=true (stage 3) → uncertain: the daemon asks, it does not commit
|
||||
// to a semantic bucket
|
||||
// - chat/query/fact+note/system → conversation/knowledge/memory_write/system
|
||||
// - act/reminder on a trusted non-action row → class "action" recorded
|
||||
// VERBATIM with illegal_action_prediction=true; never mapped to uncertain
|
||||
// - anything else → uncertain
|
||||
//
|
||||
// Reads /tmp/mvn-s22/pool.json (emit step) and writes /tmp/mvn-s22/legacy.json
|
||||
// with both the raw decision fields and the projected class, plus a summary
|
||||
// printout. No embedding is recomputed and no label is changed.
|
||||
|
||||
type poolRow struct {
|
||||
IDX int `json:"idx"`
|
||||
Text string `json:"text"`
|
||||
NText string `json:"n_text"`
|
||||
Route string `json:"route"`
|
||||
Tags []string `json:"tags"`
|
||||
CVFold int `json:"cv_fold"`
|
||||
SplitGroup string `json:"split_group"`
|
||||
FamilyID string `json:"family_id"`
|
||||
SourceID string `json:"source_id"`
|
||||
}
|
||||
|
||||
type legacyRow struct {
|
||||
IDX int `json:"idx"`
|
||||
Text string `json:"text"`
|
||||
Route string `json:"route"`
|
||||
Intent string `json:"intent"`
|
||||
Class string `json:"class"`
|
||||
Illegal bool `json:"illegal_action_prediction"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
Stage int `json:"stage"`
|
||||
Clarify bool `json:"clarify"`
|
||||
Producer string `json:"producer"`
|
||||
Error string `json:"error,omitempty"`
|
||||
SourceID string `json:"source_id"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
var mode, poolPath, outPath string
|
||||
flag.StringVar(&mode, "mode", "legacy", "baseline mode: legacy (hash classifier) or heads (ONNX cascade minus LLM)")
|
||||
flag.StringVar(&poolPath, "pool", "/tmp/mvn-s22/pool.json", "emit-step pool.json")
|
||||
flag.StringVar(&outPath, "out", "/tmp/mvn-s22/legacy.json", "output path")
|
||||
flag.Parse()
|
||||
switch mode {
|
||||
case "legacy":
|
||||
legacyMain(poolPath, outPath)
|
||||
case "heads":
|
||||
headsMain(poolPath, outPath)
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "unknown -mode %q\n", mode)
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
func legacyMain(poolPath, outPath string) {
|
||||
runOverPool(buildMinimalRouter(), poolPath, outPath)
|
||||
}
|
||||
|
||||
func runOverPool(r *router.Router, poolPath, outPath string) {
|
||||
raw, err := os.ReadFile(poolPath)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
var rows []poolRow
|
||||
if err := json.Unmarshal(raw, &rows); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
now := time.Now()
|
||||
|
||||
out := make([]legacyRow, 0, len(rows))
|
||||
classCount := map[string]int{}
|
||||
for _, pr := range rows {
|
||||
d, err := r.Route(ctx, router.NormalizedInput{Text: pr.Text}, now)
|
||||
lr := legacyRow{
|
||||
IDX: pr.IDX,
|
||||
Text: pr.Text,
|
||||
Route: pr.Route,
|
||||
SourceID: pr.SourceID,
|
||||
}
|
||||
if err != nil {
|
||||
lr.Class = "uncertain"
|
||||
lr.Error = err.Error()
|
||||
} else {
|
||||
lr.Intent = string(d.Intent)
|
||||
lr.Confidence = d.Confidence
|
||||
lr.Stage = d.Stage
|
||||
lr.Clarify = d.Clarify
|
||||
lr.Producer = string(d.Producer)
|
||||
}
|
||||
lr.Class, lr.Illegal = project(d, err)
|
||||
classCount[lr.Class]++
|
||||
out = append(out, lr)
|
||||
}
|
||||
|
||||
if err := writeJSON(outPath, out); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fmt.Printf("legacy baseline over %d residual non-action rows:\n", len(out))
|
||||
for _, c := range []string{"conversation", "knowledge", "memory_write", "system", "uncertain", "action"} {
|
||||
fmt.Printf(" %-14s %d (%.1f%%)\n", c, classCount[c], 100*float64(classCount[c])/float64(len(out)))
|
||||
}
|
||||
fmt.Printf(" illegal_action_prediction: %d\n", classCount["action"])
|
||||
// Grammar hits inside a corpus-residual population would be a
|
||||
// corpus/harness disagreement worth telling the report about: the corpus
|
||||
// marked each row not-fast-path-resolved, so a current stage-0 rule
|
||||
// resolving it means the corpus's fast-path mirror is stale or a grammar
|
||||
// landed after the corpus froze.
|
||||
gh := 0
|
||||
ghByRoute := map[string]int{}
|
||||
ghByIntent := map[string]int{}
|
||||
for _, lr := range out {
|
||||
if lr.Producer == string(router.RouteProducerGrammar) {
|
||||
gh++
|
||||
ghByRoute[lr.Route]++
|
||||
ghByIntent[lr.Intent]++
|
||||
}
|
||||
}
|
||||
fmt.Printf(" stage-0 grammar hits: %d\n", gh)
|
||||
if gh > 0 {
|
||||
fmt.Printf(" by ground-truth route: %v\n", ghByRoute)
|
||||
fmt.Printf(" by grammar intent: %v\n", ghByIntent)
|
||||
}
|
||||
}
|
||||
|
||||
// project maps the router's authoritative output into the five-way non-action
|
||||
// space, or to the "action" bucket verbatim when the router calls an act or a
|
||||
// reminder on a non-action row.
|
||||
func project(d router.Decision, err error) (string, bool) {
|
||||
if err != nil {
|
||||
return "uncertain", false
|
||||
}
|
||||
if d.Clarify {
|
||||
return "uncertain", false
|
||||
}
|
||||
switch d.Intent {
|
||||
case router.IntentChat:
|
||||
return "conversation", false
|
||||
case router.IntentQuery:
|
||||
return "knowledge", false
|
||||
case router.IntentFact, router.IntentNote:
|
||||
return "memory_write", false
|
||||
case router.IntentSystem:
|
||||
return "system", false
|
||||
case router.IntentAct, router.IntentReminder:
|
||||
return "action", true
|
||||
default:
|
||||
return "uncertain", false
|
||||
}
|
||||
}
|
||||
|
||||
func writeJSON(path string, v any) error {
|
||||
fh, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer fh.Close()
|
||||
w := bufio.NewWriter(fh)
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetIndent("", " ")
|
||||
if err := enc.Encode(v); err != nil {
|
||||
return err
|
||||
}
|
||||
return w.Flush()
|
||||
}
|
||||
Reference in New Issue
Block a user