kami
ad074cea31
Loading a different gguf was a one-line edit to phraser.model_path plus a
restart. It is now an owner-triggered IPC call, off unless configured.
internal/phraser/swap.go holds the safety properties as code:
- Never two models resident. The old llama-server is killed and reaped
before the new one is launched. One 1.7B fits the Vega iGPU; a
blue/green overlap would OOM the box, so it is not offered.
- Atomic from a turn's point of view. Swap drains the in-flight turns
(they finish on the old model), then refuses arrivals with ErrSwapping
until the new server has answered /v1/models. No turn ever sees half a
swap; refused turns fall back to the classifier cascade.
- A failed load rolls back. If the new model does not start or does not
probe, the previous one is reloaded and the call returns RolledBack
with the error. If the rollback also fails the daemon says so and
degrades to the classifier rather than pretending to serve.
Holders of the completion client are re-pointed, not rebuilt: llm.Client
guards its base URL and LLMPhraser.OnSwap re-points it, so the router, the
replier, the mail extractor and the memory evaluator follow the new port
without knowing a swap happened.
Reach is deliberately narrow. phraser.swap_models is an exact-match
allowlist of absolute paths a human wrote, rejected at startup otherwise,
so "swap the model" can never mean "load any file on my disk"; the running
model is always swappable back to. MethodSwapModel is AuthStepUp, the same
rung as mutating the tool allowlist, and /models gates POST through the
same stepUpOK the tools page uses. Nothing calls Swap on a timer and no
act, intent or utterance reaches it.
Vikunja #250
94 lines
2.7 KiB
Go
94 lines
2.7 KiB
Go
package llm
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestComplete(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != "POST" {
|
|
t.Errorf("method = %q, want POST", r.Method)
|
|
}
|
|
if !strings.HasSuffix(r.URL.Path, "/v1/chat/completions") {
|
|
t.Errorf("path = %q, want /v1/chat/completions", r.URL.Path)
|
|
}
|
|
var reqBody struct {
|
|
Messages []struct {
|
|
Role string `json:"role"`
|
|
Content string `json:"content"`
|
|
} `json:"messages"`
|
|
Grammar string `json:"grammar"`
|
|
MaxTokens int `json:"max_tokens"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&reqBody); err != nil {
|
|
t.Fatalf("decode request body: %v", err)
|
|
}
|
|
if len(reqBody.Messages) < 2 {
|
|
t.Fatalf("expected at least 2 messages, got %d", len(reqBody.Messages))
|
|
}
|
|
if reqBody.Messages[0].Role != "system" || reqBody.Messages[1].Content != "hi" {
|
|
t.Errorf("unexpected messages: %+v", reqBody.Messages)
|
|
}
|
|
if reqBody.Grammar != `root ::= "x"` {
|
|
t.Errorf("grammar = %q, want root ::= \"x\"", reqBody.Grammar)
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
|
}))
|
|
defer srv.Close()
|
|
|
|
c := New(srv.URL, 5*time.Second)
|
|
got, err := c.Complete(context.Background(), Req{
|
|
System: "be helpful",
|
|
User: "hi",
|
|
Grammar: `root ::= "x"`,
|
|
MaxTokens: 42,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Complete: %v", err)
|
|
}
|
|
if got != "ok" {
|
|
t.Errorf("got %q, want %q", got, "ok")
|
|
}
|
|
}
|
|
|
|
// TestSetBaseURL — a model swap re-points every holder of the client rather than
|
|
// rebuilding the router, the replier and the extractors (Vikunja #250).
|
|
func TestSetBaseURL(t *testing.T) {
|
|
var hit string
|
|
srvA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
hit = "A"
|
|
w.Write([]byte(`{"choices":[{"message":{"content":"a"}}]}`))
|
|
}))
|
|
defer srvA.Close()
|
|
srvB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
hit = "B"
|
|
w.Write([]byte(`{"choices":[{"message":{"content":"b"}}]}`))
|
|
}))
|
|
defer srvB.Close()
|
|
|
|
c := New(srvA.URL, 5*time.Second)
|
|
if _, err := c.Complete(context.Background(), Req{User: "x"}); err != nil {
|
|
t.Fatalf("Complete against A: %v", err)
|
|
}
|
|
if hit != "A" {
|
|
t.Fatalf("first request went to %q; want A", hit)
|
|
}
|
|
c.SetBaseURL(srvB.URL)
|
|
if got := c.BaseURL(); got != srvB.URL {
|
|
t.Errorf("BaseURL = %q; want %q", got, srvB.URL)
|
|
}
|
|
if _, err := c.Complete(context.Background(), Req{User: "x"}); err != nil {
|
|
t.Fatalf("Complete against B: %v", err)
|
|
}
|
|
if hit != "B" {
|
|
t.Errorf("request after the swap went to %q; want B", hit)
|
|
}
|
|
}
|