Maven/internal/llm/gate.go

package llm

import (
	"context"
	"sync"
	"time"
)

// Gate — priority access to the one llama-server slot.
//
// llama-server is started without -np, so it serves one request at a time and
// everything else queues. That is fine while every caller is a voice turn, and
// it stops being fine the moment a background job joins: mail extraction reads
// up to 4000 characters on a Thinking 1.7B with a two minute budget, and a turn
// that arrives during one waits for however much of that budget is left. The
// router degrades to the classifier cascade on error, so he would get the 36.8%
// floor while his mail is being read, and the phraser has no floor at all and
// simply waits.
//
// So background work asks the gate first:
//
//   - at most ONE background request is in flight, whatever the queue depth
//     upstream. A first poll of a mailbox with 40 unseen messages cannot
//     serialise 40 extractions ahead of anything.
//   - a background request waits while any foreground request is in flight, and
//     for Quiet after the last one finished. The quiet window is what stops an
//     extraction starting in the gap between the router call and the phraser
//     call of the same turn.
//
// Foreground requests never wait. This is not a fair queue and must not become
// one: the point is that the thing he is waiting for wins every time.
//
// It bounds only what goes through an *llm.Client built with SetGate. The
// phraser's own HTTP path is not gated, and a turn that reaches the phraser
// without touching the router is not marked. Every real turn routes first, so
// the marking is good enough to keep extraction out of the way; it is a
// courtesy gate, not a scheduler.
type Gate struct {
	mu sync.Mutex
	// fg — foreground requests in flight.
	fg int
	// last — when a foreground request last started or finished.
	last time.Time
	// bg — one token, so only one background request runs at a time.
	bg chan struct{}

	quiet time.Duration
	poll  time.Duration
	now   func() time.Time
}

// NewGate returns a gate that holds background work back for quiet after the
// last foreground request. quiet <= 0 means "wait only while one is in flight".
func NewGate(quiet time.Duration) *Gate {
	return &Gate{
		bg:    make(chan struct{}, 1),
		quiet: quiet,
		poll:  50 * time.Millisecond,
		now:   time.Now,
	}
}

// Foreground marks a request as the thing he is waiting for. It never blocks.
// The returned function must be called when the request finishes.
func (g *Gate) Foreground() func() {
	if g == nil {
		return func() {}
	}
	g.mu.Lock()
	g.fg++
	g.last = g.now()
	g.mu.Unlock()
	return func() {
		g.mu.Lock()
		g.fg--
		g.last = g.now()
		g.mu.Unlock()
	}
}

// AcquireBackground blocks until the slot is free enough for background work,
// or ctx is done. The returned release function must be called when the request
// finishes; it is nil on error.
func (g *Gate) AcquireBackground(ctx context.Context) (func(), error) {
	if g == nil {
		return func() {}, nil
	}
	select {
	case g.bg <- struct{}{}:
	case <-ctx.Done():
		return nil, ctx.Err()
	}
	release := func() { <-g.bg }
	for {
		if g.clear() {
			return release, nil
		}
		t := time.NewTimer(g.poll)
		select {
		case <-t.C:
		case <-ctx.Done():
			t.Stop()
			release()
			return nil, ctx.Err()
		}
	}
}

// clear reports whether no foreground request is in flight and the quiet window
// since the last one has passed.
func (g *Gate) clear() bool {
	g.mu.Lock()
	defer g.mu.Unlock()
	if g.fg > 0 {
		return false
	}
	if g.quiet <= 0 || g.last.IsZero() {
		return true
	}
	return g.now().Sub(g.last) >= g.quiet
}