Store and describe images through a shared media intake (#252)
Vision needs a second model this box does not have, so the shipped half is the part that works without one: an image arrives, is sniffed, is stored content-addressed, and is prepared for inference. The describing half is written and tested against a fake server, and refuses any endpoint that is not on this box. internal/media is the intake all three senses share — hearing and speaker recognition store their audio in the same place under the same retention. Blobs stay out of the sqlite store; only the derived text becomes a note, and only when the caller asks. Retention is enforced by an hourly prune loop rather than by a comment. The plan's RemoteProvider step is refused: no cloud model, inference stays on the box, and vision.NewLocal validates that at construction.
This commit is contained in:
kami
@@ -0,0 +1,279 @@
|
||||
// Package vision is Maven's image-understanding seam (Vikunja #252,
|
||||
// docs/plans/07-vision.md).
|
||||
//
|
||||
// One interface, Provider, with one method: describe an image, in words, in
|
||||
// Russian, with an optional question about it. Text extraction is not a second
|
||||
// method — "прочитай текст с картинки" is a prompt, and a vision-language model
|
||||
// does not have a separate OCR mode to select.
|
||||
//
|
||||
// # What is deliberately NOT here
|
||||
//
|
||||
// The plan document called for a `RemoteProvider` calling "an OpenAI-compatible
|
||||
// vision API endpoint". That step is refused: CLAUDE.md's surviving hard
|
||||
// constraint after "never phones home" was deprecated is *no cloud model,
|
||||
// inference stays on the box*, and a photo of his flat is the single worst thing
|
||||
// to make an exception for. Endpoint is therefore checked at construction and
|
||||
// must be a loopback or private address — a public host is a config error, not a
|
||||
// deployment option. That check is the reason this package does not simply reuse
|
||||
// internal/llm.Client.
|
||||
//
|
||||
// # State on this box, honestly
|
||||
//
|
||||
// The resident model is Qwen3-1.7B, which is text-only, and as of 2026-08-01
|
||||
// there is no vision-capable gguf and no mmproj file anywhere under
|
||||
// /mnt/hdd1/llms. So LocalProvider is written, tested against a fake server, and
|
||||
// currently has nothing real to talk to: the describing half is BLOCKED on a
|
||||
// model download (see docs/plans/07-vision.md for the candidates and the
|
||||
// recipe). What works today without any download is the intake — an image
|
||||
// arrives, is stored, is prepared — and the config seam that turns the rest on.
|
||||
//
|
||||
// Provider is nil-safe through Disabled, and vision is OFF unless configured,
|
||||
// like the weather and telegram.
|
||||
package vision
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/media"
|
||||
"github.com/kami/maven/internal/webfetch"
|
||||
)
|
||||
|
||||
// DefaultTimeout — budget for one description. A small VLM doing prefill over
|
||||
// an 896px image on a Vega iGPU is slow; 90s is generous because nobody is
|
||||
// holding a conversation open on this path — the answer arrives as a reply or a
|
||||
// note, and a too-tight timeout just means it never arrives at all.
|
||||
const DefaultTimeout = 90 * time.Second
|
||||
|
||||
// DefaultMaxTokens — cap on the description. A paragraph is what a spoken
|
||||
// answer can carry; a page is not.
|
||||
const DefaultMaxTokens = 300
|
||||
|
||||
// DefaultPrompt — what she is asked when he did not ask anything specific,
|
||||
// only sent a picture. Russian, because that is the channel language, and
|
||||
// feminine self-reference is not needed here (the prompt is an instruction, the
|
||||
// persona block is added by the caller that phrases the reply).
|
||||
const DefaultPrompt = "Опиши, что на этом изображении. Коротко, 2-3 предложения. Если на нём есть текст, приведи его."
|
||||
|
||||
// Errors callers distinguish.
|
||||
var (
|
||||
// ErrDisabled — vision is not configured. Returned by Disabled, which is
|
||||
// what the daemon wires when the config block is absent.
|
||||
ErrDisabled = errors.New("vision: not configured")
|
||||
// ErrNotPrivate — the configured endpoint is not on this box or its
|
||||
// network. Refused at construction; see the package comment.
|
||||
ErrNotPrivate = errors.New("vision: endpoint must be a local or private address")
|
||||
// ErrEmptyReply — the model returned nothing usable.
|
||||
ErrEmptyReply = errors.New("vision: empty description")
|
||||
)
|
||||
|
||||
// Provider — the image-understanding contract. Describe takes an image already
|
||||
// prepared by internal/media (decoded, downscaled, JPEG) and a prompt; an empty
|
||||
// prompt means DefaultPrompt.
|
||||
type Provider interface {
|
||||
Describe(ctx context.Context, im media.Image, prompt string) (string, error)
|
||||
}
|
||||
|
||||
// Disabled — the floor Provider. Every call fails with ErrDisabled, which the
|
||||
// caller turns into "я не умею смотреть картинки — зрение не настроено". It
|
||||
// exists so that no call site needs a nil check and switching vision off cannot
|
||||
// crash a turn.
|
||||
type Disabled struct{}
|
||||
|
||||
// Describe always fails. The signature matches Provider.
|
||||
func (Disabled) Describe(context.Context, media.Image, string) (string, error) {
|
||||
return "", ErrDisabled
|
||||
}
|
||||
|
||||
// Config — how to reach the local vision server. Built from
|
||||
// config.VisionConfig by the daemon; kept separate so this package does not
|
||||
// import internal/config.
|
||||
type Config struct {
|
||||
// Endpoint — base URL of a llama-server started with a vision model and its
|
||||
// mmproj (`llama-server -m model.gguf --mmproj mmproj.gguf`). Must be
|
||||
// loopback or private. The path is appended by the provider; give it
|
||||
// "http://127.0.0.1:8081".
|
||||
Endpoint string
|
||||
// Model — the model name to send. llama-server ignores it; it matters if the
|
||||
// endpoint is something else OpenAI-shaped on the same box.
|
||||
Model string
|
||||
// Timeout — per-description budget. 0 ⇒ DefaultTimeout.
|
||||
Timeout time.Duration
|
||||
// MaxTokens — cap on the reply. 0 ⇒ DefaultMaxTokens.
|
||||
MaxTokens int
|
||||
// Prompt — the default question. Empty ⇒ DefaultPrompt.
|
||||
Prompt string
|
||||
}
|
||||
|
||||
// LocalProvider talks to a llama-server on this box over its
|
||||
// /v1/chat/completions endpoint, sending the image as a data URI content part.
|
||||
// It is the only real Provider, and it is a plain HTTP client: no subprocess
|
||||
// spawning, because the daemon already owns llama-server lifecycle for the
|
||||
// resident model and a second managed process is a bigger change than this task.
|
||||
type LocalProvider struct {
|
||||
endpoint string
|
||||
model string
|
||||
prompt string
|
||||
maxTokens int
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
// NewLocal builds a LocalProvider, refusing a non-private endpoint. A bad URL
|
||||
// or a public host is an error at construction so the daemon logs it once at
|
||||
// startup instead of failing every turn.
|
||||
func NewLocal(cfg Config) (*LocalProvider, error) {
|
||||
base := strings.TrimRight(strings.TrimSpace(cfg.Endpoint), "/")
|
||||
if base == "" {
|
||||
return nil, errors.New("vision: empty endpoint")
|
||||
}
|
||||
if err := checkPrivate(base); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
timeout := cfg.Timeout
|
||||
if timeout <= 0 {
|
||||
timeout = DefaultTimeout
|
||||
}
|
||||
maxTokens := cfg.MaxTokens
|
||||
if maxTokens <= 0 {
|
||||
maxTokens = DefaultMaxTokens
|
||||
}
|
||||
prompt := strings.TrimSpace(cfg.Prompt)
|
||||
if prompt == "" {
|
||||
prompt = DefaultPrompt
|
||||
}
|
||||
return &LocalProvider{
|
||||
endpoint: base,
|
||||
model: cfg.Model,
|
||||
prompt: prompt,
|
||||
maxTokens: maxTokens,
|
||||
http: &http.Client{Timeout: timeout},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Endpoint is the server this provider talks to. For logs and /dash.
|
||||
func (p *LocalProvider) Endpoint() string { return p.endpoint }
|
||||
|
||||
// checkPrivate refuses any endpoint that is not on this box or its LAN. A
|
||||
// hostname that is not an IP literal is refused too: "vision.example.com" could
|
||||
// resolve anywhere, and resolving it here would be trusting DNS with his photos.
|
||||
// localhost is the one name allowed, because it is the common case.
|
||||
func checkPrivate(raw string) error {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("vision: parse endpoint: %w", err)
|
||||
}
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return fmt.Errorf("vision: endpoint scheme %q not supported", u.Scheme)
|
||||
}
|
||||
host := u.Hostname()
|
||||
if host == "" {
|
||||
return errors.New("vision: endpoint has no host")
|
||||
}
|
||||
if strings.EqualFold(host, "localhost") {
|
||||
return nil
|
||||
}
|
||||
ip := net.ParseIP(host)
|
||||
if ip == nil {
|
||||
return fmt.Errorf("%w: %q is a name, not an address", ErrNotPrivate, host)
|
||||
}
|
||||
if !webfetch.IsPrivateIP(ip) {
|
||||
return fmt.Errorf("%w: %s", ErrNotPrivate, host)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// chat request shapes. Content is the OpenAI multimodal array form: a text part
|
||||
// and an image_url part whose url is a data URI.
|
||||
type textPart struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
type imageURL struct {
|
||||
URL string `json:"url"`
|
||||
}
|
||||
type imagePart struct {
|
||||
Type string `json:"type"`
|
||||
ImageURL imageURL `json:"image_url"`
|
||||
}
|
||||
type chatReq struct {
|
||||
Model string `json:"model,omitempty"`
|
||||
Messages []any `json:"messages"`
|
||||
MaxTokens int `json:"max_tokens,omitempty"`
|
||||
Temp float64 `json:"temperature"`
|
||||
}
|
||||
type userMsg struct {
|
||||
Role string `json:"role"`
|
||||
Content []any `json:"content"`
|
||||
}
|
||||
type chatResp struct {
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
ReasoningContent string `json:"reasoning_content"`
|
||||
} `json:"message"`
|
||||
} `json:"choices"`
|
||||
}
|
||||
|
||||
// Describe sends the image and prompt and returns the model's answer. An empty
|
||||
// prompt uses the configured default. Errors are wrapped, never fatal: the
|
||||
// caller says she could not make out the picture and the turn continues.
|
||||
func (p *LocalProvider) Describe(ctx context.Context, im media.Image, prompt string) (string, error) {
|
||||
if len(im.JPEG) == 0 {
|
||||
return "", media.ErrEmpty
|
||||
}
|
||||
q := strings.TrimSpace(prompt)
|
||||
if q == "" {
|
||||
q = p.prompt
|
||||
}
|
||||
body, err := json.Marshal(chatReq{
|
||||
Model: p.model,
|
||||
MaxTokens: p.maxTokens,
|
||||
Messages: []any{userMsg{Role: "user", Content: []any{
|
||||
textPart{Type: "text", Text: q},
|
||||
imagePart{Type: "image_url", ImageURL: imageURL{URL: im.DataURI()}},
|
||||
}}},
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("vision: marshal: %w", err)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
p.endpoint+"/v1/chat/completions", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("vision: request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := p.http.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("vision: post: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("vision: status %d", resp.StatusCode)
|
||||
}
|
||||
var out chatResp
|
||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||
return "", fmt.Errorf("vision: decode: %w", err)
|
||||
}
|
||||
if len(out.Choices) == 0 {
|
||||
return "", ErrEmptyReply
|
||||
}
|
||||
text := strings.TrimSpace(out.Choices[0].Message.Content)
|
||||
if text == "" {
|
||||
// Same fallback as internal/llm: a Thinking model sometimes puts the
|
||||
// whole answer in reasoning_content and leaves content empty.
|
||||
text = strings.TrimSpace(out.Choices[0].Message.ReasoningContent)
|
||||
}
|
||||
if text == "" {
|
||||
return "", ErrEmptyReply
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
Reference in New Issue
Block a user