Store and describe images through a shared media intake (#252)
Vision needs a second model this box does not have, so the shipped half is the part that works without one: an image arrives, is sniffed, is stored content-addressed, and is prepared for inference. The describing half is written and tested against a fake server, and refuses any endpoint that is not on this box. internal/media is the intake all three senses share — hearing and speaker recognition store their audio in the same place under the same retention. Blobs stay out of the sqlite store; only the derived text becomes a note, and only when the caller asks. Retention is enforced by an hourly prune loop rather than by a comment. The plan's RemoteProvider step is refused: no cloud model, inference stays on the box, and vision.NewLocal validates that at construction.
This commit is contained in:
kami
@@ -0,0 +1,188 @@
|
||||
package media
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/draw"
|
||||
"image/gif"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DefaultMaxDim — the longest edge an image is scaled down to before it goes to
|
||||
// a vision model. 896 is the tile size the current crop of small
|
||||
// vision-language models (Qwen2.5-VL, SmolVLM, moondream) work in; sending a
|
||||
// 12-megapixel phone photo instead just costs the box minutes of prefill for
|
||||
// tiles that get pooled away anyway.
|
||||
const DefaultMaxDim = 896
|
||||
|
||||
// JPEGQuality for the re-encode. 85 is the usual "no visible artefacts" point,
|
||||
// and the re-encode exists to shrink the payload, not to archive it — the
|
||||
// original bytes stay in the blob store untouched.
|
||||
const JPEGQuality = 85
|
||||
|
||||
// ErrUnsupportedImage — the bytes are not an image format this build can
|
||||
// decode. Notably webp: the stdlib has no webp decoder and this repo takes no
|
||||
// new dependencies, so a webp arriving from Telegram is refused here with a
|
||||
// clear error rather than handed to a model as garbage.
|
||||
var ErrUnsupportedImage = errors.New("media: unsupported image format")
|
||||
|
||||
// SniffImage identifies image bytes by magic number and returns the mime. It
|
||||
// exists because a caller-declared content type is a claim, and the store's file
|
||||
// extension (and the vision provider's data URI) should follow the bytes.
|
||||
//
|
||||
// Returns ErrUnsupportedImage for anything unrecognised, including webp — which
|
||||
// is recognised well enough to name in the error, so the log says "webp is not
|
||||
// supported" instead of "not an image".
|
||||
func SniffImage(data []byte) (string, error) {
|
||||
switch {
|
||||
case len(data) >= 3 && data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF:
|
||||
return "image/jpeg", nil
|
||||
case len(data) >= 8 && string(data[:8]) == "\x89PNG\r\n\x1a\n":
|
||||
return "image/png", nil
|
||||
case len(data) >= 6 && (string(data[:6]) == "GIF87a" || string(data[:6]) == "GIF89a"):
|
||||
return "image/gif", nil
|
||||
case len(data) >= 12 && string(data[:4]) == "RIFF" && string(data[8:12]) == "WEBP":
|
||||
return "", fmt.Errorf("%w: webp (no decoder in this build)", ErrUnsupportedImage)
|
||||
}
|
||||
return "", ErrUnsupportedImage
|
||||
}
|
||||
|
||||
// Image — an image prepared for a vision model: JPEG bytes, downscaled, with
|
||||
// the dimensions it ended up at. It is deliberately a separate type from Blob:
|
||||
// a Blob is what he sent, an Image is what the model sees, and the two are not
|
||||
// the same bytes.
|
||||
type Image struct {
|
||||
JPEG []byte
|
||||
Width int
|
||||
Height int
|
||||
// Source names where the original came from ("telegram", "web:upload"),
|
||||
// carried through only so a log line can say what was looked at.
|
||||
Source string
|
||||
}
|
||||
|
||||
// DataURI renders the image as a `data:image/jpeg;base64,...` URI, which is how
|
||||
// every OpenAI-compatible multimodal endpoint takes an image. The string is
|
||||
// large (roughly 4/3 of the JPEG); nothing caches it.
|
||||
func (im Image) DataURI() string {
|
||||
return "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString(im.JPEG)
|
||||
}
|
||||
|
||||
// PrepareImage decodes data, scales it so its longest edge is at most maxDim
|
||||
// (never up — a small image is left alone), and re-encodes it as JPEG.
|
||||
// maxDim ≤ 0 ⇒ DefaultMaxDim.
|
||||
//
|
||||
// An image with an alpha channel is composited onto white rather than having
|
||||
// alpha dropped to black, because the common case is a screenshot or a
|
||||
// transparent-background diagram, and text on black-on-black is unreadable to
|
||||
// the model for no reason.
|
||||
func PrepareImage(data []byte, source string, maxDim int) (Image, error) {
|
||||
if len(data) == 0 {
|
||||
return Image{}, ErrEmpty
|
||||
}
|
||||
if maxDim <= 0 {
|
||||
maxDim = DefaultMaxDim
|
||||
}
|
||||
mime, err := SniffImage(data)
|
||||
if err != nil {
|
||||
return Image{}, err
|
||||
}
|
||||
src, err := decode(data, mime)
|
||||
if err != nil {
|
||||
return Image{}, fmt.Errorf("media: decode %s: %w", mime, err)
|
||||
}
|
||||
|
||||
dst := flattenAndScale(src, maxDim)
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, dst, &jpeg.Options{Quality: JPEGQuality}); err != nil {
|
||||
return Image{}, fmt.Errorf("media: encode jpeg: %w", err)
|
||||
}
|
||||
b := dst.Bounds()
|
||||
return Image{JPEG: buf.Bytes(), Width: b.Dx(), Height: b.Dy(), Source: source}, nil
|
||||
}
|
||||
|
||||
func decode(data []byte, mime string) (image.Image, error) {
|
||||
r := bytes.NewReader(data)
|
||||
switch strings.ToLower(mime) {
|
||||
case "image/jpeg":
|
||||
return jpeg.Decode(r)
|
||||
case "image/png":
|
||||
return png.Decode(r)
|
||||
case "image/gif":
|
||||
return gif.Decode(r)
|
||||
}
|
||||
return nil, ErrUnsupportedImage
|
||||
}
|
||||
|
||||
// flattenAndScale composites onto white and box-scales down to maxDim. The
|
||||
// scaler is a plain area average over the source pixels mapping to each
|
||||
// destination pixel — nearest-neighbour would alias small text into noise,
|
||||
// which defeats the point of reading a screenshot, and an area average is a
|
||||
// dozen lines against pulling in golang.org/x/image on an offline box.
|
||||
func flattenAndScale(src image.Image, maxDim int) *image.RGBA {
|
||||
sb := src.Bounds()
|
||||
sw, sh := sb.Dx(), sb.Dy()
|
||||
dw, dh := fit(sw, sh, maxDim)
|
||||
|
||||
flat := image.NewRGBA(image.Rect(0, 0, sw, sh))
|
||||
draw.Draw(flat, flat.Bounds(), image.NewUniform(image.White), image.Point{}, draw.Src)
|
||||
draw.Draw(flat, flat.Bounds(), src, sb.Min, draw.Over)
|
||||
if dw == sw && dh == sh {
|
||||
return flat
|
||||
}
|
||||
|
||||
dst := image.NewRGBA(image.Rect(0, 0, dw, dh))
|
||||
for y := 0; y < dh; y++ {
|
||||
y0, y1 := y*sh/dh, (y+1)*sh/dh
|
||||
if y1 <= y0 {
|
||||
y1 = y0 + 1
|
||||
}
|
||||
for x := 0; x < dw; x++ {
|
||||
x0, x1 := x*sw/dw, (x+1)*sw/dw
|
||||
if x1 <= x0 {
|
||||
x1 = x0 + 1
|
||||
}
|
||||
var r, g, b, n uint32
|
||||
for sy := y0; sy < y1; sy++ {
|
||||
for sx := x0; sx < x1; sx++ {
|
||||
i := flat.PixOffset(sx, sy)
|
||||
r += uint32(flat.Pix[i])
|
||||
g += uint32(flat.Pix[i+1])
|
||||
b += uint32(flat.Pix[i+2])
|
||||
n++
|
||||
}
|
||||
}
|
||||
o := dst.PixOffset(x, y)
|
||||
dst.Pix[o] = uint8(r / n)
|
||||
dst.Pix[o+1] = uint8(g / n)
|
||||
dst.Pix[o+2] = uint8(b / n)
|
||||
dst.Pix[o+3] = 0xFF
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
// fit returns the largest w×h with the same aspect ratio whose longest edge is
|
||||
// at most maxDim, never enlarging. Both edges are clamped to at least 1 so a
|
||||
// 2000×1 strip does not scale to zero height.
|
||||
func fit(w, h, maxDim int) (int, int) {
|
||||
if w <= maxDim && h <= maxDim {
|
||||
return w, h
|
||||
}
|
||||
if w >= h {
|
||||
nh := h * maxDim / w
|
||||
if nh < 1 {
|
||||
nh = 1
|
||||
}
|
||||
return maxDim, nh
|
||||
}
|
||||
nw := w * maxDim / h
|
||||
if nw < 1 {
|
||||
nw = 1
|
||||
}
|
||||
return nw, maxDim
|
||||
}
|
||||
Reference in New Issue
Block a user