Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 93c1a41d4a | |||
| bce5ed210c | |||
| c31f0d1001 |
+5
-5
@@ -206,11 +206,11 @@ func wireVoice(cfg *config.Config, coreAPI ipc.CoreAPI, phr phraser.Phraser, mem
|
||||
if threshold <= 0 {
|
||||
threshold = config.DefaultRouterThreshold
|
||||
}
|
||||
// Both routing paths are weak on held-out utterances — the classifier gets
|
||||
// 36.8% of intents right, the resident model 50.0% and much slower. Off by
|
||||
// default (see config.VoiceConfig.LLMRouter); the classifier always stays
|
||||
// wired as the fallback, so a model error never breaks a turn.
|
||||
rtr := buildRouter(emb, matcher, threshold, pickLLMRouter(cfg.Voice.LLMRouter, llmClient))
|
||||
// The resident model routes by default: 63.2% of held-out intents right
|
||||
// against the classifier's 50.0%, at about 1s a turn instead of 30ms (see
|
||||
// config.VoiceConfig.LLMRouter). The classifier always stays wired as the
|
||||
// fallback, so a model error never breaks a turn.
|
||||
rtr := buildRouter(emb, matcher, threshold, pickLLMRouter(cfg.Voice.UseLLMRouter(), llmClient))
|
||||
|
||||
// ----- sessions registry (shared with voicesink) -----
|
||||
sessions := voice.NewSessions()
|
||||
|
||||
+1
-1
@@ -40,7 +40,7 @@
|
||||
"tokenizer_path": "/opt/maven/models/embedder/multilingual-e5-small/tokenizer.json",
|
||||
"lib_path": "/opt/maven/lib/libonnxruntime.so"
|
||||
},
|
||||
"llm_router": false,
|
||||
"llm_router": true,
|
||||
"tool_timeout": "30s",
|
||||
"tools": [
|
||||
{ "name": "status", "cmd": ["systemctl", "status"], "scope": "homelab", "destructive": false },
|
||||
|
||||
+32
-10
@@ -258,18 +258,25 @@ type VoiceConfig struct {
|
||||
RouterThreshold float64 `json:"router_threshold,omitempty"`
|
||||
|
||||
// LLMRouter — route with the resident model instead of the embedding
|
||||
// classifier. Measured on the held-out fixture (ROUTING-EVAL-31-07-2026.md)
|
||||
// the model gets 50.0% of intents right against the classifier's 36.8%, but
|
||||
// it costs about 800ms per turn instead of 30ms.
|
||||
// classifier. On by default since Vikunja #320.
|
||||
//
|
||||
// TODO: the default stays false until this lands.
|
||||
// Extractor.Extract never runs on an LLM decision, so acts arrive with no
|
||||
// Fn and reminders with no Time. Turning this on today makes routing more
|
||||
// accurate and less safe.
|
||||
// Measured on the held-out fixture (ROUTING-EVAL-31-07-2026.md): 63.2% of
|
||||
// intents right against the classifier's 50.0%, and no route errors. It
|
||||
// costs about 1s per turn instead of 30ms.
|
||||
//
|
||||
// The router can now refuse: it answers "unknown" when it cannot route, and
|
||||
// the turn drops to the classifier and its clarify gate (Vikunja #359).
|
||||
LLMRouter bool `json:"llm_router,omitempty"`
|
||||
// It is safe to leave on. The model can refuse — it answers "unknown" when
|
||||
// it cannot route, and the turn drops to the classifier and its clarify
|
||||
// gate. Any LLM error does the same, so a turn never breaks on the model.
|
||||
// Slot extraction runs on LLM decisions too, so acts get their Fn and
|
||||
// reminders their Time.
|
||||
//
|
||||
// Set it false to go back to the classifier, e.g. on a box with no
|
||||
// llama-server or when 1s a turn is too slow.
|
||||
//
|
||||
// It is a pointer so that "missing from the file" and "explicitly false"
|
||||
// are different things: missing means on, false means off. Read it with
|
||||
// UseLLMRouter(), not directly.
|
||||
LLMRouter *bool `json:"llm_router,omitempty"`
|
||||
|
||||
// QueryMinScore — the note-recall confidence gate. Top cosine below this
|
||||
// ⇒ "I don't know" instead of a guess. Tuned for the ONNX embedder (0.55);
|
||||
@@ -405,6 +412,8 @@ const (
|
||||
DefaultRouterThreshold = 0.55
|
||||
DefaultQueryMinScore = 0.55
|
||||
DefaultToolTimeout = 30 * time.Second
|
||||
// DefaultLLMRouter — route with the resident model unless told otherwise.
|
||||
DefaultLLMRouter = true
|
||||
|
||||
DefaultFactEnrichmentInterval = 30 * time.Second
|
||||
)
|
||||
@@ -489,6 +498,10 @@ func (c *Config) applyDefaults() {
|
||||
if c.Voice.ToolTimeout <= 0 {
|
||||
c.Voice.ToolTimeout = Duration(DefaultToolTimeout)
|
||||
}
|
||||
if c.Voice.LLMRouter == nil {
|
||||
on := DefaultLLMRouter
|
||||
c.Voice.LLMRouter = &on
|
||||
}
|
||||
}
|
||||
|
||||
// routines: default severity to care-class (1) — the safe floor: a
|
||||
@@ -507,6 +520,15 @@ func (c *Config) applyDefaults() {
|
||||
}
|
||||
}
|
||||
|
||||
// UseLLMRouter reports whether to route with the resident model. Unset means
|
||||
// on; only an explicit false in the config turns it off.
|
||||
func (v *VoiceConfig) UseLLMRouter() bool {
|
||||
if v == nil || v.LLMRouter == nil {
|
||||
return DefaultLLMRouter
|
||||
}
|
||||
return *v.LLMRouter
|
||||
}
|
||||
|
||||
func (c *Config) validate() error {
|
||||
if c.Phraser != nil {
|
||||
if c.Phraser.ModelPath == "" {
|
||||
|
||||
@@ -171,14 +171,26 @@ func TestWeatherConfigNilOK(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMRouterDefaultsOff(t *testing.T) {
|
||||
func TestLLMRouterDefaultsOn(t *testing.T) {
|
||||
p := writeConfig(t, `{"voice":{"enabled":true,"bind":"127.0.0.1:9100"}}`)
|
||||
c, err := Load(p)
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if c.Voice.LLMRouter {
|
||||
t.Error("voice.llm_router absent should mean false")
|
||||
if !c.Voice.UseLLMRouter() {
|
||||
t.Error("voice.llm_router absent should mean on")
|
||||
}
|
||||
}
|
||||
|
||||
// Missing and explicitly false must not mean the same thing.
|
||||
func TestLLMRouterExplicitFalseTurnsItOff(t *testing.T) {
|
||||
p := writeConfig(t, `{"voice":{"enabled":true,"bind":"127.0.0.1:9100","llm_router":false}}`)
|
||||
c, err := Load(p)
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if c.Voice.UseLLMRouter() {
|
||||
t.Error("voice.llm_router false should turn it off")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -188,7 +200,7 @@ func TestLLMRouterRead(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if !c.Voice.LLMRouter {
|
||||
if !c.Voice.UseLLMRouter() {
|
||||
t.Error("voice.llm_router true was not read")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -175,3 +175,87 @@ func TestLLMRouterLLMError(t *testing.T) {
|
||||
t.Fatal("want ok=false, err!=nil on llm error")
|
||||
}
|
||||
}
|
||||
|
||||
// --- slot extraction on top of an LLM decision --------------------------------
|
||||
|
||||
// newLLMTestRouter — a router whose route always comes from the mock model.
|
||||
func newLLMTestRouter(t *testing.T, out string) *Router {
|
||||
t.Helper()
|
||||
c := NewClassifier(NewHashEmbedder(1024))
|
||||
seedClassifier(t, c)
|
||||
acts := DefaultActMatcher{Fns: []string{"restart", "stop", "run", "backup"}}
|
||||
return New(Config{
|
||||
Classifier: c,
|
||||
Extractor: Extractor{Time: StubDateTimeParser{}, Acts: acts, Facts: DefaultFactParser{}},
|
||||
Threshold: 0.4,
|
||||
LLM: NewLLMRouter(mockLLM{out: out}),
|
||||
})
|
||||
}
|
||||
|
||||
// The model cannot produce a fire time, so without extraction every LLM-routed
|
||||
// reminder was dropped as "no time".
|
||||
func TestLLMDecisionGetsReminderTime(t *testing.T) {
|
||||
r := newLLMTestRouter(t, `{"intent":"reminder","text":"позвонить маме"}`)
|
||||
d, err := r.Route(context.Background(), "напомни позвонить маме через 2 часа", refNow())
|
||||
if err != nil {
|
||||
t.Fatalf("route: %v", err)
|
||||
}
|
||||
if d.Intent != IntentReminder {
|
||||
t.Fatalf("want reminder, got %v", d.Intent)
|
||||
}
|
||||
if !d.Slots.HasTime || !d.Slots.Time.Equal(refNow().Add(2*time.Hour)) {
|
||||
t.Fatalf("want time now+2h, got %+v", d.Slots)
|
||||
}
|
||||
if d.Slots.Text != "позвонить маме" {
|
||||
t.Fatalf("extraction overwrote the model's text: %q", d.Slots.Text)
|
||||
}
|
||||
}
|
||||
|
||||
// No time in the utterance ⇒ no time in the slots. Do not invent one; the
|
||||
// daemon says it could not read the time.
|
||||
func TestLLMReminderWithoutTimeStaysEmpty(t *testing.T) {
|
||||
r := newLLMTestRouter(t, `{"intent":"reminder","text":"позвонить маме"}`)
|
||||
d, err := r.Route(context.Background(), "напомни позвонить маме", refNow())
|
||||
if err != nil {
|
||||
t.Fatalf("route: %v", err)
|
||||
}
|
||||
if d.Slots.HasTime {
|
||||
t.Fatalf("invented a time: %v", d.Slots.Time)
|
||||
}
|
||||
}
|
||||
|
||||
// An act decision arrived with no Fn, so the tool never ran.
|
||||
func TestLLMDecisionGetsActFn(t *testing.T) {
|
||||
r := newLLMTestRouter(t, `{"intent":"act","verb":"restart nginx"}`)
|
||||
d, err := r.Route(context.Background(), "слушай, restart nginx пожалуйста", refNow())
|
||||
if err != nil {
|
||||
t.Fatalf("route: %v", err)
|
||||
}
|
||||
if !d.Slots.HasFn || d.Slots.Fn != "restart" || len(d.Slots.Args) != 1 || d.Slots.Args[0] != "nginx" {
|
||||
t.Fatalf("want fn=restart args=[nginx], got %+v", d.Slots)
|
||||
}
|
||||
}
|
||||
|
||||
// The model's own slots win; extraction only fills gaps.
|
||||
func TestLLMSlotsWinOverExtraction(t *testing.T) {
|
||||
r := newLLMTestRouter(t, `{"intent":"fact","key":"hydration","value":"выпил"}`)
|
||||
d, err := r.Route(context.Background(), "я выпил воду", refNow())
|
||||
if err != nil {
|
||||
t.Fatalf("route: %v", err)
|
||||
}
|
||||
if d.Slots.Key != "hydration" {
|
||||
t.Fatalf("extraction overwrote the model's key: %q", d.Slots.Key)
|
||||
}
|
||||
}
|
||||
|
||||
// A fact the model left keyless still gets one from the parser.
|
||||
func TestLLMFactGetsKeyFromParser(t *testing.T) {
|
||||
r := newLLMTestRouter(t, `{"intent":"fact","text":"я выпил воду"}`)
|
||||
d, err := r.Route(context.Background(), "я выпил воду", refNow())
|
||||
if err != nil {
|
||||
t.Fatalf("route: %v", err)
|
||||
}
|
||||
if !d.Slots.HasKey || d.Slots.Key != "water" {
|
||||
t.Fatalf("want key=water, got %+v", d.Slots)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -88,6 +88,7 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De
|
||||
if r.llm != nil {
|
||||
if d, ok, err := r.llm.Route(ctx, utterance, now); err == nil && ok {
|
||||
d.Utterance = utterance
|
||||
r.fillSlots(ctx, &d, now)
|
||||
return d, nil
|
||||
} else if err != nil {
|
||||
log.Printf("router: llm route fell back to classifier: %v", err)
|
||||
@@ -118,6 +119,39 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// fillSlots — run stage-2 extraction on an LLM decision and fill only the slots
|
||||
// the model left empty. The LLM wins where it answered: it saw the sentence, the
|
||||
// parsers are keyword tables. Extraction covers what the model cannot produce at
|
||||
// all — a parsed reminder time and an allowlist fn.
|
||||
//
|
||||
// If a reminder still has no time, leave it missing. The daemon then says it
|
||||
// could not read the time; inventing one would set a wrong alarm.
|
||||
func (r *Router) fillSlots(ctx context.Context, d *Decision, now time.Time) {
|
||||
ex := r.extractor.Extract(ctx, d.Intent, d.Utterance, now)
|
||||
if !d.Slots.HasTime && ex.HasTime {
|
||||
d.Slots.Time, d.Slots.HasTime = ex.Time, ex.HasTime
|
||||
}
|
||||
if !d.Slots.HasKey && ex.HasKey {
|
||||
d.Slots.Key, d.Slots.Value, d.Slots.HasKey = ex.Key, ex.Value, ex.HasKey
|
||||
}
|
||||
if !d.Slots.HasFn && ex.HasFn {
|
||||
d.Slots.Fn, d.Slots.Args, d.Slots.HasFn = ex.Fn, ex.Args, ex.HasFn
|
||||
}
|
||||
// For an act the model returns the verb in Text ("restart nginx"), which is
|
||||
// often cleaner than the raw utterance ("maven, could you restart nginx").
|
||||
// Try it too when the utterance did not match the allowlist.
|
||||
if d.Intent == IntentAct && !d.Slots.HasFn && r.extractor.Acts != nil &&
|
||||
d.Slots.Text != "" && d.Slots.Text != d.Utterance {
|
||||
if fn, args, ok := r.extractor.Acts.Match(d.Slots.Text); ok {
|
||||
d.Slots.Fn, d.Slots.Args, d.Slots.HasFn = fn, args, true
|
||||
}
|
||||
}
|
||||
if d.Slots.Text == "" {
|
||||
d.Slots.Text = ex.Text
|
||||
}
|
||||
// Stage stays 1: it says who decided the route, and that was the LLM.
|
||||
}
|
||||
|
||||
// CorrectMisroute — the user corrected a bad classification. Appends a new
|
||||
// example for the corrected intent (append-only — grows the classifier, no
|
||||
// retrain). Same shape as nudges.outcome tuning cooldowns: more reliable over
|
||||
|
||||
Reference in New Issue
Block a user