diff --git a/cmd/mavgpud/auth.go b/cmd/mavgpud/auth.go new file mode 100644 index 0000000..fe2cfba --- /dev/null +++ b/cmd/mavgpud/auth.go @@ -0,0 +1,145 @@ +package main + +import ( + "crypto/sha256" + "crypto/subtle" + "fmt" + "net" + "net/http" + "os" + "strings" +) + +// The boundary in front of the card. +// +// mavgpud has to listen on the LAN, because homesrv is the client and a +// loopback default takes the model arm down. That makes this the one hop on the +// workstation anything on the network could reach, and until 2026-08-11 it +// reverse-proxied every path to llama-server unauthenticated: any client could +// spend the card, hold the model resident by touching the idle clock, and read +// /slots, which carries the prompts of whoever else was using it. +// +// So: a bearer token every request must carry, read from a file, and a path +// allowlist so a token that leaks buys the model API and not the admin one. The +// CW2 transcriber beside this daemon has worked this way since it shipped; this +// is the same arrangement, not a new one. + +// readToken loads the bearer token. The file holds the token and nothing else, +// trailing newline allowed. A path that is set and unreadable is fatal to the +// caller: a supervisor that silently ran without its boundary is the failure +// this exists to prevent. +func readToken(path string) (string, error) { + b, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("token_file: %w", err) + } + tok := strings.TrimSpace(string(b)) + if tok == "" { + return "", fmt.Errorf("token_file %s is empty", path) + } + return tok, nil +} + +// loopbackListen reports whether addr can only be reached from this machine. +// An empty or wildcard host is not loopback, which is the case that matters: +// ":8080" is the shipped default and it answers the whole LAN. +func loopbackListen(addr string) bool { + host, _, err := net.SplitHostPort(addr) + if err != nil { + host = addr + } + host = strings.Trim(host, "[]") + if host == "" { + return false + } + if host == "localhost" { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +// allowed is what a token buys. Everything llama-server exposes beyond this is +// refused, because the endpoints Maven does not call are the expensive ones to +// hand out: /slots returns other callers' prompts, and its save/restore actions +// write files chosen by the request. +// +// Adding a caller means adding its path here. That is deliberate — the list is +// short because Maven's use of the workstation is. +var allowed = map[string]string{ + "/v1/chat/completions": http.MethodPost, + "/v1/completions": http.MethodPost, + "/v1/embeddings": http.MethodPost, + "/v1/models": http.MethodGet, + "/props": http.MethodGet, +} + +// requireToken authenticates, then bounds. Order matters: an unauthenticated +// client must not be able to make this daemon allocate a body buffer. +// +// /health is not exempt. It reports whether the card is loaded and free, which +// is exactly what someone deciding whether to take it from him would ask. +func requireToken(token string, maxBody int64, next http.Handler) http.Handler { + want := sha256.Sum256([]byte(token)) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got := sha256.Sum256([]byte(bearer(r))) + if subtle.ConstantTimeCompare(got[:], want[:]) != 1 { + w.Header().Set("WWW-Authenticate", "Bearer") + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + r.Body = http.MaxBytesReader(w, r.Body, maxBody) + next.ServeHTTP(w, r) + }) +} + +// bearer pulls the credential out of the header. A malformed header yields the +// empty string, which fails the comparison like any other wrong token — there +// is no separate error for it, because telling a caller *how* it was wrong is +// the only thing a probe learns from a 401. +func bearer(r *http.Request) string { + h := r.Header.Get("Authorization") + const prefix = "Bearer " + if len(h) <= len(prefix) || !strings.EqualFold(h[:len(prefix)], prefix) { + return "" + } + return strings.TrimSpace(h[len(prefix):]) +} + +// allowlist refuses a path the model arm does not use. It answers 404 rather +// than 403 so a scan cannot map llama-server's surface through this hop. +func allowlist(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + method, ok := allowed[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + if r.Method != method { + w.Header().Set("Allow", method) + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + next.ServeHTTP(w, r) + }) +} + +// limitInflight caps concurrent proxied requests. A waiter leaves when its own +// context ends, so a client that gave up does not keep a slot: llama-server +// runs with -np 1 and queueing here is cheaper than queueing inside the child +// with a body held in memory on both sides. +func limitInflight(n int, next http.Handler) http.Handler { + if n <= 0 { + return next + } + slots := make(chan struct{}, n) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case slots <- struct{}{}: + defer func() { <-slots }() + next.ServeHTTP(w, r) + case <-r.Context().Done(): + http.Error(w, "client went away", http.StatusServiceUnavailable) + } + }) +} diff --git a/cmd/mavgpud/main.go b/cmd/mavgpud/main.go index d2cb76e..6e9c704 100644 --- a/cmd/mavgpud/main.go +++ b/cmd/mavgpud/main.go @@ -36,6 +36,21 @@ type config struct { Listen string `json:"listen"` // what Maven talks to LlamaAddr string `json:"llama_addr"` // where llama-server binds LlamaBin string `json:"llama_bin"` + + // TokenFile holds the bearer token every request must carry. It is a path + // and never the token itself, the rule mavpoll, mavmaild and the CW2 + // transcriber already follow: a secret in a committed config is a secret + // in the history. Empty is allowed only on a loopback Listen, and + // requireToken is where that is decided. + TokenFile string `json:"token_file,omitempty"` + + // MaxBody bounds a proxied request body. A completion is a prompt, and a + // prompt that does not fit here would not fit the context window either. + MaxBody int64 `json:"max_body_bytes,omitempty"` + // MaxInflight bounds how many proxied requests reach llama-server at once. + // It runs with -np 1, so anything above a handful only queues inside the + // child while holding a connection and a body in memory here. + MaxInflight int `json:"max_inflight,omitempty"` // LlamaArgs must include the flags that bind LlamaAddr. They are passed // through untouched so the model, context size and layer count stay the // owner's business and not this daemon's schema. @@ -88,6 +103,8 @@ func defaults() config { MinFreeVRAM: 15 << 30, EvictAfter: 2, StartAfter: 5, + MaxBody: 8 << 20, + MaxInflight: 4, } } @@ -123,6 +140,20 @@ func main() { log.Fatal("mavgpud: llama_bin is required") } + // A LAN listener with no token is refused rather than downgraded to + // loopback. Downgrading would look like a safe default and would take the + // model arm down instead: homesrv is the client and it is on the LAN. + var token string + if cfg.TokenFile != "" { + var err error + if token, err = readToken(cfg.TokenFile); err != nil { + log.Fatalf("mavgpud: %v", err) + } + } else if !loopbackListen(cfg.Listen) { + log.Fatalf("mavgpud: listen %s is reachable from the network and token_file is unset — "+ + "set token_file, or listen on 127.0.0.1 and accept that Maven cannot reach it", cfg.Listen) + } + base := "http://" + cfg.LlamaAddr run := newRunner("llama-server", cfg.LlamaBin, cfg.LlamaArgs, base+"/health") sup := &supervisor{ @@ -145,7 +176,20 @@ func main() { if err != nil { log.Fatalf("mavgpud: llama_addr: %v", err) } - srv := &http.Server{Addr: cfg.Listen, Handler: sup.handler(target)} + var h http.Handler = sup.handler(target) + if token != "" { + h = requireToken(token, cfg.MaxBody, h) + } + srv := &http.Server{ + Addr: cfg.Listen, + Handler: h, + // A slow-loris client holds a connection and a header buffer for free + // otherwise. No ReadTimeout or WriteTimeout: a completion legitimately + // takes minutes on this card, and either one would cut it off. + ReadHeaderTimeout: 10 * time.Second, + IdleTimeout: 60 * time.Second, + MaxHeaderBytes: 1 << 16, + } go func() { log.Printf("mavgpud: listening on %s, model %s", cfg.Listen, cfg.LlamaBin) if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { @@ -203,14 +247,14 @@ func (s *supervisor) handler(target *url.URL) http.Handler { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"status":"ok"}`)) }) - mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + mux.Handle("/", allowlist(limitInflight(s.cfg.MaxInflight, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if !s.run.isReady() { http.Error(w, "model not loaded", http.StatusServiceUnavailable) return } s.touch() proxy.ServeHTTP(w, r) - }) + })))) return mux } diff --git a/deploy/mavgpud.json b/deploy/mavgpud.json index 2738a5c..e4d272d 100644 --- a/deploy/mavgpud.json +++ b/deploy/mavgpud.json @@ -1,5 +1,14 @@ { "listen": ":8080", + "//token_file": [ + "The bearer token every request must carry. homesrv is the client and it", + "is on the LAN, so this port cannot be loopback and the token is what", + "stops anything else on the network spending the card or reading /slots.", + "A path, never the token: mavgpud refuses to start when listen is", + "reachable from the network and this is unset.", + "Same value as MAVEN_GPU_TOKEN in homesrv's deploy/telegram.env." + ], + "token_file": "/home/kami/.config/mavgpud.token", "llama_addr": "127.0.0.1:10000", "llama_bin": "llama-server", "//llama_args": [