Merge branch 'fix/g07' into fix/integrated
# Conflicts: # internal/ipc/api.go # internal/ipc/client.go # internal/llm/client.go
This commit is contained in:
+59
-5
@@ -14,14 +14,41 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// SwapGate — admission control for a completion. Enter blocks or refuses while the
|
||||
// resident model is being swapped, and the returned release says the request is
|
||||
// done. The phraser implements it: a swap kills the running llama-server, so
|
||||
// every holder of a base URL has to be counted before the kill, not just the
|
||||
// phrasing paths.
|
||||
//
|
||||
// Without this the drain saw only the phraser's own calls. The LLM router, the
|
||||
// replier, the mail extractor and the memory evaluator all reach llama-server
|
||||
// through this client, so a swap could report zero requests in flight and kill
|
||||
// the server out from under a routing decision. The turn then finished on the
|
||||
// new model, which is the "half of one model and half of another" the swap is
|
||||
// supposed to make impossible.
|
||||
//
|
||||
// Distinct from Gate, which is about priority between a voice turn and a
|
||||
// background job. This one is about the model underneath them changing. A
|
||||
// request passes the priority gate first and this one second, so nothing waits
|
||||
// for a quiet window while counted as in flight against the drain.
|
||||
type SwapGate interface {
|
||||
Enter() (release func(), err error)
|
||||
}
|
||||
|
||||
type Client struct {
|
||||
// mu guards base only. The base URL changes when the daemon swaps the
|
||||
// resident model (Vikunja #250): llama-server is relaunched on a fresh
|
||||
// port, and every holder of this client — the LLM router, the replier, the
|
||||
// mail extractor — must follow without being rebuilt. One mutexed field is
|
||||
// the whole mechanism; a swap re-points the client, it does not replace it.
|
||||
// mu guards base and gate. The base URL can change when the daemon swaps the
|
||||
// resident model (Vikunja #250) and every holder of this client — the LLM
|
||||
// router, the replier, the mail extractor — must follow without being
|
||||
// rebuilt. A swap re-points the client, it does not replace it.
|
||||
//
|
||||
// On the deploy shape the new server binds the same fixed port the killed
|
||||
// one released (startLlamaProc passes the port out of phraser.listen), so
|
||||
// SetBaseURL is normally a no-op and the gate is the part doing the work.
|
||||
// The re-pointing stays because nothing guarantees the port: a phraser
|
||||
// listening on :0, or a future swap that moves the server, changes the base.
|
||||
mu sync.RWMutex
|
||||
base string
|
||||
swap SwapGate
|
||||
http *http.Client
|
||||
|
||||
// gate / background — priority on the single llama-server slot. Set once
|
||||
@@ -51,6 +78,24 @@ func (c *Client) gateFor() (*Gate, bool) {
|
||||
return c.gate, c.background
|
||||
}
|
||||
|
||||
// SetSwapGate installs the swap admission gate. Nil (the default, and what the
|
||||
// eval harness and the tests use) means no gating.
|
||||
func (c *Client) SetSwapGate(g SwapGate) {
|
||||
c.mu.Lock()
|
||||
c.swap = g
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
func (c *Client) enter() (func(), error) {
|
||||
c.mu.RLock()
|
||||
g := c.swap
|
||||
c.mu.RUnlock()
|
||||
if g == nil {
|
||||
return func() {}, nil
|
||||
}
|
||||
return g.Enter()
|
||||
}
|
||||
|
||||
func New(baseURL string, timeout time.Duration) *Client {
|
||||
return &Client{base: baseURL, http: &http.Client{Timeout: timeout}}
|
||||
}
|
||||
@@ -107,6 +152,8 @@ type resp struct {
|
||||
}
|
||||
|
||||
func (c *Client) Complete(ctx context.Context, r Req) (string, error) {
|
||||
// Priority first: a background request can sit here for a while, and it
|
||||
// must not be counted against the swap drain while it waits.
|
||||
if g, background := c.gateFor(); g != nil {
|
||||
if background {
|
||||
release, err := g.AcquireBackground(ctx)
|
||||
@@ -118,6 +165,13 @@ func (c *Client) Complete(ctx context.Context, r Req) (string, error) {
|
||||
defer g.Foreground()()
|
||||
}
|
||||
}
|
||||
// Then the swap drain, which counts what is actually about to hit the
|
||||
// server it is going to kill.
|
||||
release, err := c.enter()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
b, _ := json.Marshal(body{
|
||||
Messages: []msg{{Role: "system", Content: r.System}, {Role: "user", Content: r.User}},
|
||||
MaxTokens: r.MaxTokens,
|
||||
|
||||
Reference in New Issue
Block a user