llm: prefer the workstation model, floor on the resident one (V-485)
Pair holds both models and decides which answers. A prober asks the remote whether it will take work and caches the answer, so a request reads an atomic bool rather than paying for a health check. Routing sits at p50 825ms on the hot path and must never wait on a machine that may be asleep. The two methods are the two halves of the degradation rule in docs/offload.md. Complete falls back silently, for routing, replies and nudge phrasing, where the big model is only better. CompleteRemote returns ErrRemoteUnavailable instead, for a world question, where the 1.7B does not answer worse but invents. A nil remote is the unconfigured deploy: nothing probes, everything goes to the floor, and the box behaves exactly as it does today.
This commit is contained in:
@@ -0,0 +1,193 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Pair — a preferred model on another host, with the resident one as the floor.
|
||||
//
|
||||
// homesrv cannot grow a GPU and the workstation has 16GB of VRAM, so the big
|
||||
// model runs there and the resident Qwen3-1.7B stays here. See docs/offload.md.
|
||||
// The workstation is never assumed up: its GPU is often busy with CPT runs and
|
||||
// the manga-recap pipeline, and the machine sleeps. So the remote is preferred,
|
||||
// never required, and Pair is what makes "preferred" mean something precise.
|
||||
//
|
||||
// This is admission control, not a scheduler. There is no arbiter deciding who
|
||||
// gets the card. A prober asks the remote whether it will take work, caches the
|
||||
// answer, and every request reads that cached answer in nanoseconds. Routing
|
||||
// sits on the hot path at p50 825ms and must never wait on a machine that may
|
||||
// be asleep, so no request ever pays for a health check itself.
|
||||
//
|
||||
// Pair satisfies nothing by itself. Callers pick a method by which half of the
|
||||
// degradation rule they live under:
|
||||
//
|
||||
// - Complete falls back silently. For routing, replies, and nudge phrasing,
|
||||
// where the big model is only better and the 1.7B is today's shipping
|
||||
// quality. He is not told which model phrased his reply.
|
||||
// - CompleteRemote returns ErrRemoteUnavailable instead of falling back. For
|
||||
// a world question, or a long Kiwix or search passage, where a 1.7B
|
||||
// confabulates rather than summarises. A named gap beats an invented
|
||||
// answer.
|
||||
type Pair struct {
|
||||
remote *Client
|
||||
floor *Client
|
||||
|
||||
// up — the cached admission answer, written only by the prober goroutine
|
||||
// and read by every request. Atomic so the read costs nanoseconds and no
|
||||
// request ever contends with the prober.
|
||||
up atomic.Bool
|
||||
|
||||
health string
|
||||
interval time.Duration
|
||||
http *http.Client
|
||||
stop chan struct{}
|
||||
}
|
||||
|
||||
// ErrRemoteUnavailable — the workstation model was required and is not
|
||||
// answering. Callers on the naming half of the degradation rule turn this into
|
||||
// a gap in the reply ("не могу сейчас"), never into a guess from the floor.
|
||||
var ErrRemoteUnavailable = errors.New("llm: workstation model unavailable")
|
||||
|
||||
// ErrNoFloor — a Pair was built with no resident model to fall back to. A
|
||||
// configuration mistake: the floor is the whole point.
|
||||
var ErrNoFloor = errors.New("llm: no floor client")
|
||||
|
||||
// NewPair builds the two-model arrangement. remote may be nil, which is the
|
||||
// unconfigured deploy and must behave exactly as the box behaves today: every
|
||||
// call goes to the floor and nothing probes anything.
|
||||
//
|
||||
// health is the URL the prober asks. llama-server's /health answers "is a model
|
||||
// loaded and ready", which is the useful signal here, because llama-server
|
||||
// refuses to load at all when VRAM is short. That makes a busy card detectable
|
||||
// without any cooperation from the owner's other jobs.
|
||||
func NewPair(remote, floor *Client, health string, interval time.Duration) *Pair {
|
||||
p := &Pair{
|
||||
remote: remote,
|
||||
floor: floor,
|
||||
health: health,
|
||||
interval: interval,
|
||||
http: &http.Client{Timeout: probeTimeout},
|
||||
stop: make(chan struct{}),
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// probeTimeout — a remote that cannot answer /health this fast is not going to
|
||||
// serve a turn either. Short on purpose: the prober runs on its own goroutine,
|
||||
// but a slow probe still delays the moment Maven notices the card came back.
|
||||
const probeTimeout = 2 * time.Second
|
||||
|
||||
// Start begins probing. It returns immediately, and the first probe runs before
|
||||
// the first tick so a remote that is already up is used on the first turn
|
||||
// rather than after one interval of falling back. Safe to call with a nil
|
||||
// remote; it does nothing.
|
||||
func (p *Pair) Start(ctx context.Context) {
|
||||
if p.remote == nil || p.health == "" {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
p.probe(ctx)
|
||||
t := time.NewTicker(p.interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-p.stop:
|
||||
return
|
||||
case <-t.C:
|
||||
p.probe(ctx)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Stop ends the prober. Idempotent.
|
||||
func (p *Pair) Stop() {
|
||||
select {
|
||||
case <-p.stop:
|
||||
default:
|
||||
close(p.stop)
|
||||
}
|
||||
}
|
||||
|
||||
// Available reports whether the workstation will take work right now. It reads
|
||||
// a cached flag, so it is safe to call per turn on the hot path. A false answer
|
||||
// is never stale in the direction that matters: the worst case is that Maven
|
||||
// falls back for up to one probe interval after the card frees up.
|
||||
func (p *Pair) Available() bool {
|
||||
return p.remote != nil && p.up.Load()
|
||||
}
|
||||
|
||||
func (p *Pair) probe(ctx context.Context) {
|
||||
ctx, cancel := context.WithTimeout(ctx, probeTimeout)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.health, nil)
|
||||
if err != nil {
|
||||
p.set(false)
|
||||
return
|
||||
}
|
||||
resp, err := p.http.Do(req)
|
||||
if err != nil {
|
||||
p.set(false)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
p.set(resp.StatusCode == http.StatusOK)
|
||||
}
|
||||
|
||||
// set records the admission answer and logs only the transitions. A machine
|
||||
// that sleeps every night would otherwise write one line per interval forever.
|
||||
func (p *Pair) set(up bool) {
|
||||
if p.up.Swap(up) == up {
|
||||
return
|
||||
}
|
||||
if up {
|
||||
log.Printf("llm: workstation model available at %s", p.health)
|
||||
} else {
|
||||
log.Printf("llm: workstation model unavailable, falling back to the resident model")
|
||||
}
|
||||
}
|
||||
|
||||
// Complete runs r on the workstation when it will take work, and on the
|
||||
// resident model otherwise. A remote that fails mid-request falls back too: the
|
||||
// admission answer is a cache and can be one interval out of date, so an error
|
||||
// here is expected rather than exceptional.
|
||||
//
|
||||
// This is the silent half of the degradation rule. It must be indistinguishable
|
||||
// from today's behaviour when the workstation is down.
|
||||
func (p *Pair) Complete(ctx context.Context, r Req) (string, error) {
|
||||
if p.floor == nil {
|
||||
return "", ErrNoFloor
|
||||
}
|
||||
if p.Available() {
|
||||
out, err := p.remote.Complete(ctx, r)
|
||||
if err == nil {
|
||||
return out, nil
|
||||
}
|
||||
// The cached answer was wrong. Correct it now rather than sending the
|
||||
// next request into the same hole, then fall back.
|
||||
p.set(false)
|
||||
}
|
||||
return p.floor.Complete(ctx, r)
|
||||
}
|
||||
|
||||
// CompleteRemote runs r on the workstation or refuses. It never falls back,
|
||||
// because for a world question the resident 1.7B does not answer worse, it
|
||||
// invents. Callers turn ErrRemoteUnavailable into a named gap.
|
||||
func (p *Pair) CompleteRemote(ctx context.Context, r Req) (string, error) {
|
||||
if !p.Available() {
|
||||
return "", ErrRemoteUnavailable
|
||||
}
|
||||
out, err := p.remote.Complete(ctx, r)
|
||||
if err != nil {
|
||||
p.set(false)
|
||||
return "", errors.Join(ErrRemoteUnavailable, err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
Reference in New Issue
Block a user