package llm import ( "context" "sync" "time" ) // Gate — priority access to the one llama-server slot. // // llama-server is started without -np, so it serves one request at a time and // everything else queues. That is fine while every caller is a voice turn, and // it stops being fine the moment a background job joins: mail extraction reads // up to 4000 characters on a Thinking 1.7B with a two minute budget, and a turn // that arrives during one waits for however much of that budget is left. The // router degrades to the classifier cascade on error, so he would get the 36.8% // floor while his mail is being read, and the phraser has no floor at all and // simply waits. // // So background work asks the gate first: // // - at most ONE background request is in flight, whatever the queue depth // upstream. A first poll of a mailbox with 40 unseen messages cannot // serialise 40 extractions ahead of anything. // - a background request waits while any foreground request is in flight, and // for Quiet after the last one finished. The quiet window is what stops an // extraction starting in the gap between the router call and the phraser // call of the same turn. // // Foreground requests never wait. This is not a fair queue and must not become // one: the point is that the thing he is waiting for wins every time. // // It bounds only what goes through an *llm.Client built with SetGate. The // phraser's own HTTP path is not gated, and a turn that reaches the phraser // without touching the router is not marked. Every real turn routes first, so // the marking is good enough to keep extraction out of the way; it is a // courtesy gate, not a scheduler. type Gate struct { mu sync.Mutex // fg — foreground requests in flight. fg int // last — when a foreground request last started or finished. last time.Time // bg — one token, so only one background request runs at a time. bg chan struct{} quiet time.Duration poll time.Duration now func() time.Time } // NewGate returns a gate that holds background work back for quiet after the // last foreground request. quiet <= 0 means "wait only while one is in flight". func NewGate(quiet time.Duration) *Gate { return &Gate{ bg: make(chan struct{}, 1), quiet: quiet, poll: 50 * time.Millisecond, now: time.Now, } } // Foreground marks a request as the thing he is waiting for. It never blocks. // The returned function must be called when the request finishes. func (g *Gate) Foreground() func() { if g == nil { return func() {} } g.mu.Lock() g.fg++ g.last = g.now() g.mu.Unlock() return func() { g.mu.Lock() g.fg-- g.last = g.now() g.mu.Unlock() } } // AcquireBackground blocks until the slot is free enough for background work, // or ctx is done. The returned release function must be called when the request // finishes; it is nil on error. func (g *Gate) AcquireBackground(ctx context.Context) (func(), error) { if g == nil { return func() {}, nil } select { case g.bg <- struct{}{}: case <-ctx.Done(): return nil, ctx.Err() } release := func() { <-g.bg } for { if g.clear() { return release, nil } t := time.NewTimer(g.poll) select { case <-t.C: case <-ctx.Done(): t.Stop() release() return nil, ctx.Err() } } } // clear reports whether no foreground request is in flight and the quiet window // since the last one has passed. func (g *Gate) clear() bool { g.mu.Lock() defer g.mu.Unlock() if g.fg > 0 { return false } if g.quiet <= 0 || g.last.IsZero() { return true } return g.now().Sub(g.last) >= g.quiet }