llm: give voice turns priority on the single llama-server slot

llama-server is started without -np, so it serves one request at a time and everything else queues. Mail extraction is allowed two minutes on a Thinking 1.7B, and the reader hands core up to 25 messages back to back. A turn arriving mid-extraction therefore waited for whatever was left of that budget: the router timed out into the classifier cascade and its 36.8% floor, and the phraser, which has no floor, simply waited. Memory evaluation had the same shape with a five minute budget. llm.Gate is the bound. Foreground requests never wait. Background requests run one at a time and yield while a foreground request is in flight, plus a quiet window after it that covers the gap between the router call and the phraser call of one turn. Clients get their priority from llmClientFor or llmBackgroundClientFor, so which side a caller is on is decided at wiring time. It gates only what goes through those clients, which the comment on Gate says. mail intake: the extraction timeout no longer wraps the capture writes. A model answering at 119 seconds of a 120 second budget left the first CaptureTask one second and the third none, so candidates the model had already produced were dropped with a deadline error. The mailbox name is validated before it becomes provenance, since "email:" is not a source and neither is an arbitrary string posted at the socket. The enable log prints the normalised candidate bound rather than the configured one, which said "max 0" and then wrote three. Found in review of #64. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TrVSBKe3RFDF4fGYKWYQnX
2026-08-01 14:05:07 +04:00
parent 6c81df17ec
commit aee20a6abc
9 changed files with 443 additions and 11 deletions
@@ -23,6 +23,32 @@ type Client struct {
 	mu   sync.RWMutex
 	base string
 	http *http.Client
+
+	// gate / background — priority on the single llama-server slot. Set once
+	// at wiring time (SetGate), read on every request. nil gate ⇒ no gating,
+	// which is what every test and every non-daemon caller gets.
+	gate       *Gate
+	background bool
+}
+
+// SetGate gives this client a priority on the shared llama-server slot. Call it
+// immediately after New, before the client is handed to anything: the fields are
+// read under the same lock as base, but the intent is one-time wiring, not a
+// knob to turn at runtime.
+//
+// background = false means "he is waiting for this" and never blocks.
+// background = true means the request yields to voice turns and runs one at a
+// time. See Gate.
+func (c *Client) SetGate(g *Gate, background bool) {
+	c.mu.Lock()
+	c.gate, c.background = g, background
+	c.mu.Unlock()
+}
+
+func (c *Client) gateFor() (*Gate, bool) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.gate, c.background
 }

 func New(baseURL string, timeout time.Duration) *Client {
@@ -81,6 +107,17 @@ type resp struct {
 }

 func (c *Client) Complete(ctx context.Context, r Req) (string, error) {
+	if g, background := c.gateFor(); g != nil {
+		if background {
+			release, err := g.AcquireBackground(ctx)
+			if err != nil {
+				return "", err
+			}
+			defer release()
+		} else {
+			defer g.Foreground()()
+		}
+	}
 	b, _ := json.Marshal(body{
 		Messages:      []msg{{Role: "system", Content: r.System}, {Role: "user", Content: r.User}},
 		MaxTokens:     r.MaxTokens,
@@ -0,0 +1,121 @@
+package llm
+
+import (
+	"context"
+	"sync"
+	"time"
+)
+
+// Gate — priority access to the one llama-server slot.
+//
+// llama-server is started without -np, so it serves one request at a time and
+// everything else queues. That is fine while every caller is a voice turn, and
+// it stops being fine the moment a background job joins: mail extraction reads
+// up to 4000 characters on a Thinking 1.7B with a two minute budget, and a turn
+// that arrives during one waits for however much of that budget is left. The
+// router degrades to the classifier cascade on error, so he would get the 36.8%
+// floor while his mail is being read, and the phraser has no floor at all and
+// simply waits.
+//
+// So background work asks the gate first:
+//
+//   - at most ONE background request is in flight, whatever the queue depth
+//     upstream. A first poll of a mailbox with 40 unseen messages cannot
+//     serialise 40 extractions ahead of anything.
+//   - a background request waits while any foreground request is in flight, and
+//     for Quiet after the last one finished. The quiet window is what stops an
+//     extraction starting in the gap between the router call and the phraser
+//     call of the same turn.
+//
+// Foreground requests never wait. This is not a fair queue and must not become
+// one: the point is that the thing he is waiting for wins every time.
+//
+// It bounds only what goes through an *llm.Client built with SetGate. The
+// phraser's own HTTP path is not gated, and a turn that reaches the phraser
+// without touching the router is not marked. Every real turn routes first, so
+// the marking is good enough to keep extraction out of the way; it is a
+// courtesy gate, not a scheduler.
+type Gate struct {
+	mu sync.Mutex
+	// fg — foreground requests in flight.
+	fg int
+	// last — when a foreground request last started or finished.
+	last time.Time
+	// bg — one token, so only one background request runs at a time.
+	bg chan struct{}
+
+	quiet time.Duration
+	poll  time.Duration
+	now   func() time.Time
+}
+
+// NewGate returns a gate that holds background work back for quiet after the
+// last foreground request. quiet <= 0 means "wait only while one is in flight".
+func NewGate(quiet time.Duration) *Gate {
+	return &Gate{
+		bg:    make(chan struct{}, 1),
+		quiet: quiet,
+		poll:  50 * time.Millisecond,
+		now:   time.Now,
+	}
+}
+
+// Foreground marks a request as the thing he is waiting for. It never blocks.
+// The returned function must be called when the request finishes.
+func (g *Gate) Foreground() func() {
+	if g == nil {
+		return func() {}
+	}
+	g.mu.Lock()
+	g.fg++
+	g.last = g.now()
+	g.mu.Unlock()
+	return func() {
+		g.mu.Lock()
+		g.fg--
+		g.last = g.now()
+		g.mu.Unlock()
+	}
+}
+
+// AcquireBackground blocks until the slot is free enough for background work,
+// or ctx is done. The returned release function must be called when the request
+// finishes; it is nil on error.
+func (g *Gate) AcquireBackground(ctx context.Context) (func(), error) {
+	if g == nil {
+		return func() {}, nil
+	}
+	select {
+	case g.bg <- struct{}{}:
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+	release := func() { <-g.bg }
+	for {
+		if g.clear() {
+			return release, nil
+		}
+		t := time.NewTimer(g.poll)
+		select {
+		case <-t.C:
+		case <-ctx.Done():
+			t.Stop()
+			release()
+			return nil, ctx.Err()
+		}
+	}
+}
+
+// clear reports whether no foreground request is in flight and the quiet window
+// since the last one has passed.
+func (g *Gate) clear() bool {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	if g.fg > 0 {
+		return false
+	}
+	if g.quiet <= 0 || g.last.IsZero() {
+		return true
+	}
+	return g.now().Sub(g.last) >= g.quiet
+}
@@ -0,0 +1,101 @@
+package llm
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+// Background work must not start while he is waiting on a turn. llama-server
+// serves one request at a time, so an extraction that starts first holds the
+// slot for its whole budget.
+func TestGateBackgroundWaitsForForeground(t *testing.T) {
+	g := NewGate(0)
+	g.poll = time.Millisecond
+	done := g.Foreground()
+
+	started := make(chan struct{})
+	go func() {
+		release, err := g.AcquireBackground(context.Background())
+		if err != nil {
+			t.Errorf("acquire: %v", err)
+			return
+		}
+		close(started)
+		release()
+	}()
+
+	select {
+	case <-started:
+		t.Fatal("background work started while a foreground request was in flight")
+	case <-time.After(20 * time.Millisecond):
+	}
+	done()
+	select {
+	case <-started:
+	case <-time.After(time.Second):
+		t.Fatal("background work never started after the foreground request finished")
+	}
+}
+
+// Only one background request at a time, whatever the queue depth upstream. A
+// first poll of a mailbox with 40 unseen messages must not put 40 extractions
+// on the slot.
+func TestGateOneBackgroundAtATime(t *testing.T) {
+	g := NewGate(0)
+	g.poll = time.Millisecond
+	first, err := g.AcquireBackground(context.Background())
+	if err != nil {
+		t.Fatalf("first: %v", err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	if _, err := g.AcquireBackground(ctx); err == nil {
+		t.Fatal("a second background request ran alongside the first")
+	}
+	first()
+	second, err := g.AcquireBackground(context.Background())
+	if err != nil {
+		t.Fatalf("second after release: %v", err)
+	}
+	second()
+}
+
+// The quiet window covers the gap between the router call and the phraser call
+// of one turn, so an extraction cannot slip in mid-turn.
+func TestGateQuietWindow(t *testing.T) {
+	now := time.Now()
+	g := NewGate(time.Minute)
+	g.poll = time.Millisecond
+	g.now = func() time.Time { return now }
+	g.Foreground()()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	if _, err := g.AcquireBackground(ctx); err == nil {
+		t.Fatal("background work started inside the quiet window")
+	}
+	now = now.Add(2 * time.Minute)
+	release, err := g.AcquireBackground(context.Background())
+	if err != nil {
+		t.Fatalf("acquire after the quiet window: %v", err)
+	}
+	release()
+}
+
+// Foreground never waits, whatever else is in flight.
+func TestGateForegroundNeverBlocks(t *testing.T) {
+	g := NewGate(time.Minute)
+	release, err := g.AcquireBackground(context.Background())
+	if err != nil {
+		t.Fatalf("acquire: %v", err)
+	}
+	defer release()
+	done := make(chan struct{})
+	go func() { g.Foreground()(); close(done) }()
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("a foreground request waited behind background work")
+	}
+}