Store and describe images through a shared media intake (#252)
Vision needs a second model this box does not have, so the shipped half is the part that works without one: an image arrives, is sniffed, is stored content-addressed, and is prepared for inference. The describing half is written and tested against a fake server, and refuses any endpoint that is not on this box. internal/media is the intake all three senses share — hearing and speaker recognition store their audio in the same place under the same retention. Blobs stay out of the sqlite store; only the derived text becomes a note, and only when the caller asks. Retention is enforced by an hourly prune loop rather than by a comment. The plan's RemoteProvider step is refused: no cloud model, inference stays on the box, and vision.NewLocal validates that at construction.
This commit is contained in:
kami
@@ -0,0 +1,111 @@
|
||||
package vision
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/kami/maven/internal/media"
|
||||
)
|
||||
|
||||
// Intake is the whole path from "bytes arrived" to "here is what she saw",
|
||||
// in one place, so that every surface that can receive an image — a Telegram
|
||||
// photo, a mavweb upload, a file path he names — goes through the same steps in
|
||||
// the same order:
|
||||
//
|
||||
// 1. sniff the bytes (the sender's declared content type is not trusted);
|
||||
// 2. store them content-addressed, so the same photo twice is one file and the
|
||||
// original is still on disk if the description came out wrong;
|
||||
// 3. prepare a downscaled JPEG for the model;
|
||||
// 4. describe it.
|
||||
//
|
||||
// Step 2 happens BEFORE step 4 deliberately. If the vision model is missing or
|
||||
// broken — which is today's actual state on this box — the image is still safely
|
||||
// stored and describable later, and the failure is "I can't look at it yet", not
|
||||
// "it's gone".
|
||||
//
|
||||
// Writing the description as a note is NOT done here. That needs the store and
|
||||
// the embedder and belongs to the daemon; Intake returns the text and lets the
|
||||
// caller decide whether it becomes a note, a reply, or both.
|
||||
type Intake struct {
|
||||
store *media.Store
|
||||
provider Provider
|
||||
maxDim int
|
||||
}
|
||||
|
||||
// NewIntake wires an intake. provider may be Disabled — storing still works,
|
||||
// which is the point. maxDim ≤ 0 ⇒ media.DefaultMaxDim.
|
||||
func NewIntake(store *media.Store, provider Provider, maxDim int) *Intake {
|
||||
if provider == nil {
|
||||
provider = Disabled{}
|
||||
}
|
||||
return &Intake{store: store, provider: provider, maxDim: maxDim}
|
||||
}
|
||||
|
||||
// Result — what an intake produced. Blob is always set when Store succeeded, so
|
||||
// a caller that got an error from the description still knows what was kept and
|
||||
// can retry against the same id later.
|
||||
type Result struct {
|
||||
Blob media.Blob
|
||||
Image media.Image
|
||||
Description string
|
||||
}
|
||||
|
||||
// Accept stores data and describes it. source is provenance recorded on the
|
||||
// blob ("telegram", "web:upload"); question is what he asked about the image, or
|
||||
// empty for the default "what is this".
|
||||
//
|
||||
// A description failure is returned alongside a populated Result: the caller
|
||||
// gets the blob id for the log and the reply, and the error to explain why there
|
||||
// are no words yet.
|
||||
func (in *Intake) Accept(ctx context.Context, data []byte, source, question string) (Result, error) {
|
||||
if in == nil || in.store == nil {
|
||||
return Result{}, fmt.Errorf("vision: intake not wired")
|
||||
}
|
||||
mime, err := media.SniffImage(data)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
blob, err := in.store.Put(media.KindImage, mime, source, data)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
im, err := media.PrepareImage(data, source, in.maxDim)
|
||||
if err != nil {
|
||||
return Result{Blob: blob}, err
|
||||
}
|
||||
res := Result{Blob: blob, Image: im}
|
||||
text, err := in.provider.Describe(ctx, im, question)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
res.Description = strings.TrimSpace(text)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// Rerun describes an already-stored image again — a different question, or the
|
||||
// first successful attempt after the model finally landed on disk. It is the
|
||||
// reason step 2 comes before step 4.
|
||||
func (in *Intake) Rerun(ctx context.Context, id, question string) (Result, error) {
|
||||
if in == nil || in.store == nil {
|
||||
return Result{}, fmt.Errorf("vision: intake not wired")
|
||||
}
|
||||
blob, data, err := in.store.Read(id)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if blob.Kind != media.KindImage {
|
||||
return Result{Blob: blob}, fmt.Errorf("vision: %s is %s, not an image", id[:12], blob.Kind)
|
||||
}
|
||||
im, err := media.PrepareImage(data, blob.Source, in.maxDim)
|
||||
if err != nil {
|
||||
return Result{Blob: blob}, err
|
||||
}
|
||||
res := Result{Blob: blob, Image: im}
|
||||
text, err := in.provider.Describe(ctx, im, question)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
res.Description = strings.TrimSpace(text)
|
||||
return res, nil
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package vision
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"image"
|
||||
"image/png"
|
||||
"testing"
|
||||
|
||||
"github.com/kami/maven/internal/media"
|
||||
)
|
||||
|
||||
type fakeProvider struct {
|
||||
reply string
|
||||
err error
|
||||
seen int
|
||||
lastQ string
|
||||
lastDim int
|
||||
}
|
||||
|
||||
func (f *fakeProvider) Describe(_ context.Context, im media.Image, prompt string) (string, error) {
|
||||
f.seen++
|
||||
f.lastQ = prompt
|
||||
f.lastDim = im.Width
|
||||
return f.reply, f.err
|
||||
}
|
||||
|
||||
func pngPayload(t *testing.T, w, h int) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
if err := png.Encode(&buf, image.NewRGBA(image.Rect(0, 0, w, h))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func testIntake(t *testing.T, p Provider) (*Intake, *media.Store) {
|
||||
t.Helper()
|
||||
s, err := media.Open(t.TempDir(), 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return NewIntake(s, p, 64), s
|
||||
}
|
||||
|
||||
func TestAcceptStoresThenDescribes(t *testing.T) {
|
||||
fp := &fakeProvider{reply: "кот на подоконнике"}
|
||||
in, store := testIntake(t, fp)
|
||||
|
||||
res, err := in.Accept(context.Background(), pngPayload(t, 200, 100), "telegram", "кто это?")
|
||||
if err != nil {
|
||||
t.Fatalf("accept: %v", err)
|
||||
}
|
||||
if res.Description != "кот на подоконнике" {
|
||||
t.Errorf("description = %q", res.Description)
|
||||
}
|
||||
if fp.lastQ != "кто это?" {
|
||||
t.Errorf("question not passed through: %q", fp.lastQ)
|
||||
}
|
||||
if fp.lastDim != 64 {
|
||||
t.Errorf("image not downscaled to maxDim: width %d", fp.lastDim)
|
||||
}
|
||||
// The sniffed mime wins over anything a sender claimed.
|
||||
got, _, err := store.Read(res.Blob.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("blob not stored: %v", err)
|
||||
}
|
||||
if got.MIME != "image/png" || got.Source != "telegram" {
|
||||
t.Errorf("blob metadata = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The ordering promise: with no vision model on the box — today's real state —
|
||||
// the image is still on disk and the id is still reported, so it can be
|
||||
// described later instead of being lost.
|
||||
func TestAcceptKeepsBlobWhenDescribeFails(t *testing.T) {
|
||||
in, store := testIntake(t, Disabled{})
|
||||
res, err := in.Accept(context.Background(), pngPayload(t, 32, 32), "web:upload", "")
|
||||
if !errors.Is(err, ErrDisabled) {
|
||||
t.Fatalf("got %v, want ErrDisabled", err)
|
||||
}
|
||||
if res.Blob.ID == "" {
|
||||
t.Fatal("no blob id reported on a description failure")
|
||||
}
|
||||
if _, _, err := store.Read(res.Blob.ID); err != nil {
|
||||
t.Errorf("blob was not kept: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRerunDescribesAStoredBlob(t *testing.T) {
|
||||
fp := &fakeProvider{reply: "текст: ошибка E24"}
|
||||
in, _ := testIntake(t, fp)
|
||||
first, err := in.Accept(context.Background(), pngPayload(t, 40, 40), "telegram", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res, err := in.Rerun(context.Background(), first.Blob.ID, "прочитай текст")
|
||||
if err != nil {
|
||||
t.Fatalf("rerun: %v", err)
|
||||
}
|
||||
if res.Description != "текст: ошибка E24" {
|
||||
t.Errorf("description = %q", res.Description)
|
||||
}
|
||||
if fp.lastQ != "прочитай текст" {
|
||||
t.Errorf("new question not used: %q", fp.lastQ)
|
||||
}
|
||||
if fp.seen != 2 {
|
||||
t.Errorf("provider called %d times, want 2", fp.seen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRerunRefusesAudioBlob(t *testing.T) {
|
||||
in, store := testIntake(t, &fakeProvider{reply: "x"})
|
||||
b, err := store.Put(media.KindAudio, "audio/wav", "capture:meeting", []byte("pcm bytes"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := in.Rerun(context.Background(), b.ID, ""); err == nil {
|
||||
t.Error("audio blob was accepted as an image")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRerunUnknownID(t *testing.T) {
|
||||
in, _ := testIntake(t, &fakeProvider{})
|
||||
if _, err := in.Rerun(context.Background(), "nope", ""); err == nil {
|
||||
t.Error("malformed id accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptRefusesNonImage(t *testing.T) {
|
||||
in, _ := testIntake(t, &fakeProvider{})
|
||||
if _, err := in.Accept(context.Background(), []byte("this is a text file"), "web:upload", ""); !errors.Is(err, media.ErrUnsupportedImage) {
|
||||
t.Errorf("got %v, want ErrUnsupportedImage", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNilProviderDegradesToDisabled(t *testing.T) {
|
||||
s, err := media.Open(t.TempDir(), 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
in := NewIntake(s, nil, 0)
|
||||
if _, err := in.Accept(context.Background(), pngPayload(t, 8, 8), "x", ""); !errors.Is(err, ErrDisabled) {
|
||||
t.Errorf("got %v, want ErrDisabled", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
// Package vision is Maven's image-understanding seam (Vikunja #252,
|
||||
// docs/plans/07-vision.md).
|
||||
//
|
||||
// One interface, Provider, with one method: describe an image, in words, in
|
||||
// Russian, with an optional question about it. Text extraction is not a second
|
||||
// method — "прочитай текст с картинки" is a prompt, and a vision-language model
|
||||
// does not have a separate OCR mode to select.
|
||||
//
|
||||
// # What is deliberately NOT here
|
||||
//
|
||||
// The plan document called for a `RemoteProvider` calling "an OpenAI-compatible
|
||||
// vision API endpoint". That step is refused: CLAUDE.md's surviving hard
|
||||
// constraint after "never phones home" was deprecated is *no cloud model,
|
||||
// inference stays on the box*, and a photo of his flat is the single worst thing
|
||||
// to make an exception for. Endpoint is therefore checked at construction and
|
||||
// must be a loopback or private address — a public host is a config error, not a
|
||||
// deployment option. That check is the reason this package does not simply reuse
|
||||
// internal/llm.Client.
|
||||
//
|
||||
// # State on this box, honestly
|
||||
//
|
||||
// The resident model is Qwen3-1.7B, which is text-only, and as of 2026-08-01
|
||||
// there is no vision-capable gguf and no mmproj file anywhere under
|
||||
// /mnt/hdd1/llms. So LocalProvider is written, tested against a fake server, and
|
||||
// currently has nothing real to talk to: the describing half is BLOCKED on a
|
||||
// model download (see docs/plans/07-vision.md for the candidates and the
|
||||
// recipe). What works today without any download is the intake — an image
|
||||
// arrives, is stored, is prepared — and the config seam that turns the rest on.
|
||||
//
|
||||
// Provider is nil-safe through Disabled, and vision is OFF unless configured,
|
||||
// like the weather and telegram.
|
||||
package vision
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/media"
|
||||
"github.com/kami/maven/internal/webfetch"
|
||||
)
|
||||
|
||||
// DefaultTimeout — budget for one description. A small VLM doing prefill over
|
||||
// an 896px image on a Vega iGPU is slow; 90s is generous because nobody is
|
||||
// holding a conversation open on this path — the answer arrives as a reply or a
|
||||
// note, and a too-tight timeout just means it never arrives at all.
|
||||
const DefaultTimeout = 90 * time.Second
|
||||
|
||||
// DefaultMaxTokens — cap on the description. A paragraph is what a spoken
|
||||
// answer can carry; a page is not.
|
||||
const DefaultMaxTokens = 300
|
||||
|
||||
// DefaultPrompt — what she is asked when he did not ask anything specific,
|
||||
// only sent a picture. Russian, because that is the channel language, and
|
||||
// feminine self-reference is not needed here (the prompt is an instruction, the
|
||||
// persona block is added by the caller that phrases the reply).
|
||||
const DefaultPrompt = "Опиши, что на этом изображении. Коротко, 2-3 предложения. Если на нём есть текст, приведи его."
|
||||
|
||||
// Errors callers distinguish.
|
||||
var (
|
||||
// ErrDisabled — vision is not configured. Returned by Disabled, which is
|
||||
// what the daemon wires when the config block is absent.
|
||||
ErrDisabled = errors.New("vision: not configured")
|
||||
// ErrNotPrivate — the configured endpoint is not on this box or its
|
||||
// network. Refused at construction; see the package comment.
|
||||
ErrNotPrivate = errors.New("vision: endpoint must be a local or private address")
|
||||
// ErrEmptyReply — the model returned nothing usable.
|
||||
ErrEmptyReply = errors.New("vision: empty description")
|
||||
)
|
||||
|
||||
// Provider — the image-understanding contract. Describe takes an image already
|
||||
// prepared by internal/media (decoded, downscaled, JPEG) and a prompt; an empty
|
||||
// prompt means DefaultPrompt.
|
||||
type Provider interface {
|
||||
Describe(ctx context.Context, im media.Image, prompt string) (string, error)
|
||||
}
|
||||
|
||||
// Disabled — the floor Provider. Every call fails with ErrDisabled, which the
|
||||
// caller turns into "я не умею смотреть картинки — зрение не настроено". It
|
||||
// exists so that no call site needs a nil check and switching vision off cannot
|
||||
// crash a turn.
|
||||
type Disabled struct{}
|
||||
|
||||
// Describe always fails. The signature matches Provider.
|
||||
func (Disabled) Describe(context.Context, media.Image, string) (string, error) {
|
||||
return "", ErrDisabled
|
||||
}
|
||||
|
||||
// Config — how to reach the local vision server. Built from
|
||||
// config.VisionConfig by the daemon; kept separate so this package does not
|
||||
// import internal/config.
|
||||
type Config struct {
|
||||
// Endpoint — base URL of a llama-server started with a vision model and its
|
||||
// mmproj (`llama-server -m model.gguf --mmproj mmproj.gguf`). Must be
|
||||
// loopback or private. The path is appended by the provider; give it
|
||||
// "http://127.0.0.1:8081".
|
||||
Endpoint string
|
||||
// Model — the model name to send. llama-server ignores it; it matters if the
|
||||
// endpoint is something else OpenAI-shaped on the same box.
|
||||
Model string
|
||||
// Timeout — per-description budget. 0 ⇒ DefaultTimeout.
|
||||
Timeout time.Duration
|
||||
// MaxTokens — cap on the reply. 0 ⇒ DefaultMaxTokens.
|
||||
MaxTokens int
|
||||
// Prompt — the default question. Empty ⇒ DefaultPrompt.
|
||||
Prompt string
|
||||
}
|
||||
|
||||
// LocalProvider talks to a llama-server on this box over its
|
||||
// /v1/chat/completions endpoint, sending the image as a data URI content part.
|
||||
// It is the only real Provider, and it is a plain HTTP client: no subprocess
|
||||
// spawning, because the daemon already owns llama-server lifecycle for the
|
||||
// resident model and a second managed process is a bigger change than this task.
|
||||
type LocalProvider struct {
|
||||
endpoint string
|
||||
model string
|
||||
prompt string
|
||||
maxTokens int
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
// NewLocal builds a LocalProvider, refusing a non-private endpoint. A bad URL
|
||||
// or a public host is an error at construction so the daemon logs it once at
|
||||
// startup instead of failing every turn.
|
||||
func NewLocal(cfg Config) (*LocalProvider, error) {
|
||||
base := strings.TrimRight(strings.TrimSpace(cfg.Endpoint), "/")
|
||||
if base == "" {
|
||||
return nil, errors.New("vision: empty endpoint")
|
||||
}
|
||||
if err := checkPrivate(base); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
timeout := cfg.Timeout
|
||||
if timeout <= 0 {
|
||||
timeout = DefaultTimeout
|
||||
}
|
||||
maxTokens := cfg.MaxTokens
|
||||
if maxTokens <= 0 {
|
||||
maxTokens = DefaultMaxTokens
|
||||
}
|
||||
prompt := strings.TrimSpace(cfg.Prompt)
|
||||
if prompt == "" {
|
||||
prompt = DefaultPrompt
|
||||
}
|
||||
return &LocalProvider{
|
||||
endpoint: base,
|
||||
model: cfg.Model,
|
||||
prompt: prompt,
|
||||
maxTokens: maxTokens,
|
||||
http: &http.Client{Timeout: timeout},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Endpoint is the server this provider talks to. For logs and /dash.
|
||||
func (p *LocalProvider) Endpoint() string { return p.endpoint }
|
||||
|
||||
// checkPrivate refuses any endpoint that is not on this box or its LAN. A
|
||||
// hostname that is not an IP literal is refused too: "vision.example.com" could
|
||||
// resolve anywhere, and resolving it here would be trusting DNS with his photos.
|
||||
// localhost is the one name allowed, because it is the common case.
|
||||
func checkPrivate(raw string) error {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("vision: parse endpoint: %w", err)
|
||||
}
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return fmt.Errorf("vision: endpoint scheme %q not supported", u.Scheme)
|
||||
}
|
||||
host := u.Hostname()
|
||||
if host == "" {
|
||||
return errors.New("vision: endpoint has no host")
|
||||
}
|
||||
if strings.EqualFold(host, "localhost") {
|
||||
return nil
|
||||
}
|
||||
ip := net.ParseIP(host)
|
||||
if ip == nil {
|
||||
return fmt.Errorf("%w: %q is a name, not an address", ErrNotPrivate, host)
|
||||
}
|
||||
if !webfetch.IsPrivateIP(ip) {
|
||||
return fmt.Errorf("%w: %s", ErrNotPrivate, host)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// chat request shapes. Content is the OpenAI multimodal array form: a text part
|
||||
// and an image_url part whose url is a data URI.
|
||||
type textPart struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
type imageURL struct {
|
||||
URL string `json:"url"`
|
||||
}
|
||||
type imagePart struct {
|
||||
Type string `json:"type"`
|
||||
ImageURL imageURL `json:"image_url"`
|
||||
}
|
||||
type chatReq struct {
|
||||
Model string `json:"model,omitempty"`
|
||||
Messages []any `json:"messages"`
|
||||
MaxTokens int `json:"max_tokens,omitempty"`
|
||||
Temp float64 `json:"temperature"`
|
||||
}
|
||||
type userMsg struct {
|
||||
Role string `json:"role"`
|
||||
Content []any `json:"content"`
|
||||
}
|
||||
type chatResp struct {
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
ReasoningContent string `json:"reasoning_content"`
|
||||
} `json:"message"`
|
||||
} `json:"choices"`
|
||||
}
|
||||
|
||||
// Describe sends the image and prompt and returns the model's answer. An empty
|
||||
// prompt uses the configured default. Errors are wrapped, never fatal: the
|
||||
// caller says she could not make out the picture and the turn continues.
|
||||
func (p *LocalProvider) Describe(ctx context.Context, im media.Image, prompt string) (string, error) {
|
||||
if len(im.JPEG) == 0 {
|
||||
return "", media.ErrEmpty
|
||||
}
|
||||
q := strings.TrimSpace(prompt)
|
||||
if q == "" {
|
||||
q = p.prompt
|
||||
}
|
||||
body, err := json.Marshal(chatReq{
|
||||
Model: p.model,
|
||||
MaxTokens: p.maxTokens,
|
||||
Messages: []any{userMsg{Role: "user", Content: []any{
|
||||
textPart{Type: "text", Text: q},
|
||||
imagePart{Type: "image_url", ImageURL: imageURL{URL: im.DataURI()}},
|
||||
}}},
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("vision: marshal: %w", err)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
p.endpoint+"/v1/chat/completions", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("vision: request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := p.http.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("vision: post: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("vision: status %d", resp.StatusCode)
|
||||
}
|
||||
var out chatResp
|
||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||
return "", fmt.Errorf("vision: decode: %w", err)
|
||||
}
|
||||
if len(out.Choices) == 0 {
|
||||
return "", ErrEmptyReply
|
||||
}
|
||||
text := strings.TrimSpace(out.Choices[0].Message.Content)
|
||||
if text == "" {
|
||||
// Same fallback as internal/llm: a Thinking model sometimes puts the
|
||||
// whole answer in reasoning_content and leaves content empty.
|
||||
text = strings.TrimSpace(out.Choices[0].Message.ReasoningContent)
|
||||
}
|
||||
if text == "" {
|
||||
return "", ErrEmptyReply
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package vision
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"image"
|
||||
"image/png"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/media"
|
||||
)
|
||||
|
||||
func testImage(t *testing.T) media.Image {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
if err := png.Encode(&buf, image.NewRGBA(image.Rect(0, 0, 32, 32))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
im, err := media.PrepareImage(buf.Bytes(), "test", 32)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return im
|
||||
}
|
||||
|
||||
func TestDisabledAlwaysRefuses(t *testing.T) {
|
||||
_, err := Disabled{}.Describe(context.Background(), testImage(t), "что тут?")
|
||||
if !errors.Is(err, ErrDisabled) {
|
||||
t.Fatalf("got %v, want ErrDisabled", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The whole reason this package has its own HTTP client instead of reusing
|
||||
// internal/llm.Client: a vision endpoint that is not on this box is refused.
|
||||
func TestNewLocalRefusesNonPrivateEndpoints(t *testing.T) {
|
||||
bad := []string{
|
||||
"https://api.openai.com",
|
||||
"http://8.8.8.8:8080",
|
||||
"https://vision.example.com", // a name could resolve anywhere
|
||||
"ftp://127.0.0.1:8080", // wrong scheme
|
||||
"", // nothing to talk to
|
||||
}
|
||||
for _, ep := range bad {
|
||||
if _, err := NewLocal(Config{Endpoint: ep}); err == nil {
|
||||
t.Errorf("NewLocal(%q) was accepted", ep)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewLocalAcceptsLocalEndpoints(t *testing.T) {
|
||||
for _, ep := range []string{"http://127.0.0.1:8081", "http://localhost:8081/", "http://192.168.1.104:8081", "http://[::1]:8081"} {
|
||||
p, err := NewLocal(Config{Endpoint: ep})
|
||||
if err != nil {
|
||||
t.Errorf("NewLocal(%q): %v", ep, err)
|
||||
continue
|
||||
}
|
||||
if strings.HasSuffix(p.Endpoint(), "/") {
|
||||
t.Errorf("trailing slash kept: %q", p.Endpoint())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescribeSendsImageAsDataURIAndReturnsText(t *testing.T) {
|
||||
var gotBody map[string]any
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v1/chat/completions" {
|
||||
t.Errorf("path = %s", r.URL.Path)
|
||||
}
|
||||
raw, _ := io.ReadAll(r.Body)
|
||||
if err := json.Unmarshal(raw, &gotBody); err != nil {
|
||||
t.Errorf("unmarshal request: %v", err)
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":" На картинке кот "}}]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p, err := NewLocal(Config{Endpoint: srv.URL, Model: "qwen-vl"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text, err := p.Describe(context.Background(), testImage(t), "кто на фото?")
|
||||
if err != nil {
|
||||
t.Fatalf("describe: %v", err)
|
||||
}
|
||||
if text != "На картинке кот" {
|
||||
t.Errorf("text = %q (should be trimmed)", text)
|
||||
}
|
||||
|
||||
msgs, ok := gotBody["messages"].([]any)
|
||||
if !ok || len(msgs) != 1 {
|
||||
t.Fatalf("messages = %#v", gotBody["messages"])
|
||||
}
|
||||
parts, ok := msgs[0].(map[string]any)["content"].([]any)
|
||||
if !ok || len(parts) != 2 {
|
||||
t.Fatalf("content parts = %#v", msgs[0])
|
||||
}
|
||||
if got := parts[0].(map[string]any)["text"]; got != "кто на фото?" {
|
||||
t.Errorf("prompt = %v", got)
|
||||
}
|
||||
url := parts[1].(map[string]any)["image_url"].(map[string]any)["url"].(string)
|
||||
if !strings.HasPrefix(url, "data:image/jpeg;base64,") {
|
||||
t.Errorf("image not sent as a jpeg data uri: %.40s", url)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescribeUsesDefaultPromptWhenNoQuestion(t *testing.T) {
|
||||
var sentPrompt string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
Messages []struct {
|
||||
Content []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
} `json:"messages"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
sentPrompt = body.Messages[0].Content[0].Text
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ок"}}]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p, err := NewLocal(Config{Endpoint: srv.URL, Prompt: "Опиши по-русски."})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := p.Describe(context.Background(), testImage(t), " "); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sentPrompt != "Опиши по-русски." {
|
||||
t.Errorf("prompt = %q", sentPrompt)
|
||||
}
|
||||
}
|
||||
|
||||
// A Thinking model sometimes leaves content empty and puts the answer in
|
||||
// reasoning_content; internal/llm has the same fallback and vision needs it too.
|
||||
func TestDescribeFallsBackToReasoningContent(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"","reasoning_content":"схема платы"}}]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
p, _ := NewLocal(Config{Endpoint: srv.URL})
|
||||
text, err := p.Describe(context.Background(), testImage(t), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if text != "схема платы" {
|
||||
t.Errorf("text = %q", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescribeErrors(t *testing.T) {
|
||||
t.Run("no choices", func(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"choices":[]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
p, _ := NewLocal(Config{Endpoint: srv.URL})
|
||||
if _, err := p.Describe(context.Background(), testImage(t), ""); !errors.Is(err, ErrEmptyReply) {
|
||||
t.Errorf("got %v, want ErrEmptyReply", err)
|
||||
}
|
||||
})
|
||||
t.Run("server error", func(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer srv.Close()
|
||||
p, _ := NewLocal(Config{Endpoint: srv.URL})
|
||||
if _, err := p.Describe(context.Background(), testImage(t), ""); err == nil {
|
||||
t.Error("500 was not an error")
|
||||
}
|
||||
})
|
||||
t.Run("empty image", func(t *testing.T) {
|
||||
p, _ := NewLocal(Config{Endpoint: "http://127.0.0.1:1"})
|
||||
if _, err := p.Describe(context.Background(), media.Image{}, ""); !errors.Is(err, media.ErrEmpty) {
|
||||
t.Errorf("got %v, want media.ErrEmpty", err)
|
||||
}
|
||||
})
|
||||
t.Run("context cancelled", func(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"поздно"}}]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
p, _ := NewLocal(Config{Endpoint: srv.URL})
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
|
||||
defer cancel()
|
||||
if _, err := p.Describe(ctx, testImage(t), ""); err == nil {
|
||||
t.Error("cancelled context returned no error")
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user