package vision import ( "context" "fmt" "strings" "github.com/kami/maven/internal/media" ) // Intake is the whole path from "bytes arrived" to "here is what she saw", // in one place, so that every surface that can receive an image — a Telegram // photo, a mavweb upload, a file path he names — goes through the same steps in // the same order: // // 1. sniff the bytes (the sender's declared content type is not trusted); // 2. store them content-addressed, so the same photo twice is one file and the // original is still on disk if the description came out wrong; // 3. prepare a downscaled JPEG for the model; // 4. describe it. // // Step 2 happens BEFORE step 4 deliberately. If the vision model is missing or // broken — which is today's actual state on this box — the image is still safely // stored and describable later, and the failure is "I can't look at it yet", not // "it's gone". // // Writing the description as a note is NOT done here. That needs the store and // the embedder and belongs to the daemon; Intake returns the text and lets the // caller decide whether it becomes a note, a reply, or both. type Intake struct { store *media.Store provider Provider maxDim int } // NewIntake wires an intake. provider may be Disabled — storing still works, // which is the point. maxDim ≤ 0 ⇒ media.DefaultMaxDim. func NewIntake(store *media.Store, provider Provider, maxDim int) *Intake { if provider == nil { provider = Disabled{} } return &Intake{store: store, provider: provider, maxDim: maxDim} } // Result — what an intake produced. Blob is always set when Store succeeded, so // a caller that got an error from the description still knows what was kept and // can retry against the same id later. type Result struct { Blob media.Blob Image media.Image Description string } // Accept stores data and describes it. source is provenance recorded on the // blob ("telegram", "web:upload"); question is what he asked about the image, or // empty for the default "what is this". // // A description failure is returned alongside a populated Result: the caller // gets the blob id for the log and the reply, and the error to explain why there // are no words yet. func (in *Intake) Accept(ctx context.Context, data []byte, source, question string) (Result, error) { if in == nil || in.store == nil { return Result{}, fmt.Errorf("vision: intake not wired") } mime, err := media.SniffImage(data) if err != nil { return Result{}, err } blob, err := in.store.Put(media.KindImage, mime, source, data) if err != nil { return Result{}, err } im, err := media.PrepareImage(data, source, in.maxDim) if err != nil { return Result{Blob: blob}, err } res := Result{Blob: blob, Image: im} text, err := in.provider.Describe(ctx, im, question) if err != nil { return res, err } res.Description = strings.TrimSpace(text) return res, nil } // Rerun describes an already-stored image again — a different question, or the // first successful attempt after the model finally landed on disk. It is the // reason step 2 comes before step 4. func (in *Intake) Rerun(ctx context.Context, id, question string) (Result, error) { if in == nil || in.store == nil { return Result{}, fmt.Errorf("vision: intake not wired") } blob, data, err := in.store.Read(id) if err != nil { return Result{}, err } if blob.Kind != media.KindImage { return Result{Blob: blob}, fmt.Errorf("vision: %s is %s, not an image", id[:12], blob.Kind) } im, err := media.PrepareImage(data, blob.Source, in.maxDim) if err != nil { return Result{Blob: blob}, err } res := Result{Blob: blob, Image: im} text, err := in.provider.Describe(ctx, im, question) if err != nil { return res, err } res.Description = strings.TrimSpace(text) return res, nil }