kami
d92349ca6e
Vision needs a second model this box does not have, so the shipped half is the part that works without one: an image arrives, is sniffed, is stored content-addressed, and is prepared for inference. The describing half is written and tested against a fake server, and refuses any endpoint that is not on this box. internal/media is the intake all three senses share — hearing and speaker recognition store their audio in the same place under the same retention. Blobs stay out of the sqlite store; only the derived text becomes a note, and only when the caller asks. Retention is enforced by an hourly prune loop rather than by a comment. The plan's RemoteProvider step is refused: no cloud model, inference stays on the box, and vision.NewLocal validates that at construction.
112 lines
3.7 KiB
Go
112 lines
3.7 KiB
Go
package vision
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/kami/maven/internal/media"
|
|
)
|
|
|
|
// Intake is the whole path from "bytes arrived" to "here is what she saw",
|
|
// in one place, so that every surface that can receive an image — a Telegram
|
|
// photo, a mavweb upload, a file path he names — goes through the same steps in
|
|
// the same order:
|
|
//
|
|
// 1. sniff the bytes (the sender's declared content type is not trusted);
|
|
// 2. store them content-addressed, so the same photo twice is one file and the
|
|
// original is still on disk if the description came out wrong;
|
|
// 3. prepare a downscaled JPEG for the model;
|
|
// 4. describe it.
|
|
//
|
|
// Step 2 happens BEFORE step 4 deliberately. If the vision model is missing or
|
|
// broken — which is today's actual state on this box — the image is still safely
|
|
// stored and describable later, and the failure is "I can't look at it yet", not
|
|
// "it's gone".
|
|
//
|
|
// Writing the description as a note is NOT done here. That needs the store and
|
|
// the embedder and belongs to the daemon; Intake returns the text and lets the
|
|
// caller decide whether it becomes a note, a reply, or both.
|
|
type Intake struct {
|
|
store *media.Store
|
|
provider Provider
|
|
maxDim int
|
|
}
|
|
|
|
// NewIntake wires an intake. provider may be Disabled — storing still works,
|
|
// which is the point. maxDim ≤ 0 ⇒ media.DefaultMaxDim.
|
|
func NewIntake(store *media.Store, provider Provider, maxDim int) *Intake {
|
|
if provider == nil {
|
|
provider = Disabled{}
|
|
}
|
|
return &Intake{store: store, provider: provider, maxDim: maxDim}
|
|
}
|
|
|
|
// Result — what an intake produced. Blob is always set when Store succeeded, so
|
|
// a caller that got an error from the description still knows what was kept and
|
|
// can retry against the same id later.
|
|
type Result struct {
|
|
Blob media.Blob
|
|
Image media.Image
|
|
Description string
|
|
}
|
|
|
|
// Accept stores data and describes it. source is provenance recorded on the
|
|
// blob ("telegram", "web:upload"); question is what he asked about the image, or
|
|
// empty for the default "what is this".
|
|
//
|
|
// A description failure is returned alongside a populated Result: the caller
|
|
// gets the blob id for the log and the reply, and the error to explain why there
|
|
// are no words yet.
|
|
func (in *Intake) Accept(ctx context.Context, data []byte, source, question string) (Result, error) {
|
|
if in == nil || in.store == nil {
|
|
return Result{}, fmt.Errorf("vision: intake not wired")
|
|
}
|
|
mime, err := media.SniffImage(data)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
blob, err := in.store.Put(media.KindImage, mime, source, data)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
im, err := media.PrepareImage(data, source, in.maxDim)
|
|
if err != nil {
|
|
return Result{Blob: blob}, err
|
|
}
|
|
res := Result{Blob: blob, Image: im}
|
|
text, err := in.provider.Describe(ctx, im, question)
|
|
if err != nil {
|
|
return res, err
|
|
}
|
|
res.Description = strings.TrimSpace(text)
|
|
return res, nil
|
|
}
|
|
|
|
// Rerun describes an already-stored image again — a different question, or the
|
|
// first successful attempt after the model finally landed on disk. It is the
|
|
// reason step 2 comes before step 4.
|
|
func (in *Intake) Rerun(ctx context.Context, id, question string) (Result, error) {
|
|
if in == nil || in.store == nil {
|
|
return Result{}, fmt.Errorf("vision: intake not wired")
|
|
}
|
|
blob, data, err := in.store.Read(id)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
if blob.Kind != media.KindImage {
|
|
return Result{Blob: blob}, fmt.Errorf("vision: %s is %s, not an image", id[:12], blob.Kind)
|
|
}
|
|
im, err := media.PrepareImage(data, blob.Source, in.maxDim)
|
|
if err != nil {
|
|
return Result{Blob: blob}, err
|
|
}
|
|
res := Result{Blob: blob, Image: im}
|
|
text, err := in.provider.Describe(ctx, im, question)
|
|
if err != nil {
|
|
return res, err
|
|
}
|
|
res.Description = strings.TrimSpace(text)
|
|
return res, nil
|
|
}
|