// mavend/vision.go — core's half of image understanding (Vikunja #252, // docs/plans/07-vision.md). // // The split: any surface that can receive a picture (mavweb upload, a Telegram // photo through mavpoll, a path he names) hands the bytes to core over // ipc.MethodDescribeImage. Core stores them content-addressed under // media.dir, prepares a downscaled JPEG, and asks a local vision server what it // is. The description comes back as words; nothing about the image is echoed. // // Off unless configured: no `media` block ⇒ nowhere to keep the bytes, so the // method does not exist and a surface cannot make Maven accept a photo by // merely sending one. A `media` block with no `vision` block is a real state, // the one this box is in today: the store is wired, the method exists, the // bytes are kept and the reply says she cannot read the picture yet. That reply // is re-runnable by id on the day a vision model lands, which is the reason to // keep the bytes at all. Saving the description as a note needs more than the // read rung — see the scope check on auth.ImageNoteSource. // // Two things this file deliberately does not do: // // - No cloud vision call, ever. internal/vision refuses a non-private // endpoint at construction; there is no config shape here that could reach // an upstream API even if someone wanted one. // - No automatic memory. SaveNote is opt-in per call. Glancing at a screenshot // is not the same act as remembering it, and a 1.7B-class VLM's guess about // a photo is not a fact worth carrying around. package main import ( "context" "errors" "fmt" "log" "path/filepath" "sync" "time" "github.com/kami/maven/internal/config" "github.com/kami/maven/internal/ipc" "github.com/kami/maven/internal/media" "github.com/kami/maven/internal/router" "github.com/kami/maven/internal/store" "github.com/kami/maven/internal/vision" ) // prunePeriod — how often stored blobs are checked against media.retention. // Hourly is far more often than needed for a 7-day retention and costs a // directory walk over a handful of sidecars; the point is that the promise is // kept by a loop that runs, not by an operator remembering a cron. const prunePeriod = time.Hour // mediaKeeper — the blob store plus the loop that enforces its retention. The // two are one object because a store without the loop is a directory that grows // forever, and shipping that would break the only interesting promise this // capability makes. type mediaKeeper struct { store *media.Store } // openMediaStore builds the blob store from config, or returns nil when media is // not configured. A relative dir resolves against StateDir, the same rule the db // and socket paths follow. func openMediaStore(cfg *config.Config) *mediaKeeper { dir := cfg.Media.StoreDir() if dir == "" { return nil } if !filepath.IsAbs(dir) && cfg.StateDir != "" { dir = filepath.Join(cfg.StateDir, dir) } st, err := media.OpenWithBudget(dir, cfg.Media.MaxBytes, cfg.Media.MaxTotalBytes, time.Duration(cfg.Media.Retention)) if err != nil { log.Printf("media: %v — image and audio intake disabled", err) return nil } log.Printf("media: blob store at %s, retention %s, %d of %d bytes used", st.Dir(), st.Retention(), st.Total(), st.Budget()) return &mediaKeeper{store: st} } // runPrune deletes over-retention blobs on a loop until ctx ends. It prunes once // immediately, so a daemon restarted after a long downtime does not sit on a // month of stale recordings until the first tick. func (k *mediaKeeper) runPrune(ctx context.Context) { prune := func() { n, err := k.store.Prune() if err != nil { log.Printf("media: prune: %v", err) return } if n > 0 { log.Printf("media: pruned %d blob(s) older than %s", n, k.store.Retention()) } } prune() t := time.NewTicker(prunePeriod) defer t.Stop() for { select { case <-ctx.Done(): return case <-t.C: prune() } } } // visionIntake — one image at a time: store, prepare, describe, optionally note. type visionIntake struct { in *vision.Intake st *store.Store emb router.Embedder now func() time.Time } // newVisionIntake returns nil when there is nothing to wire. keeper == nil means // no media block, which disables the method outright; a missing or disabled // vision block still wires the method, because storing an image and answering // "I can't look at it yet" is more useful than pretending the surface does not // exist — and it is exactly the state this box is in until a vision model is on // disk. func newVisionIntake(keeper *mediaKeeper, st *store.Store, emb router.Embedder, cfg *config.Config) *visionIntake { if keeper == nil { return nil } vc := cfg.Vision maxDim := 0 var provider vision.Provider = vision.Disabled{} if vc.LooksAtImages() { p, err := vision.NewLocal(vision.Config{ Endpoint: vc.Endpoint, Model: vc.Model, Timeout: time.Duration(vc.Timeout), MaxTokens: vc.MaxTokens, Prompt: vc.Prompt, }) if err != nil { // A public endpoint, a hostname, a bad URL. Logged once here rather // than failing every turn, and the store still works. log.Printf("vision: %v — she can store images but not describe them", err) } else { provider = p maxDim = vc.MaxDim log.Printf("vision: enabled against %s", p.Endpoint()) } } else { log.Printf("vision: not configured — images are stored, not described") } return &visionIntake{ in: vision.NewIntake(keeper.store, provider, maxDim), st: st, emb: emb, now: time.Now, } } // describe handles one ipc.MethodDescribeImage call. // // A description failure is NOT an error out of this method when the bytes were // stored: the caller gets the id and an empty description, which is honest ("it // is kept, I cannot read it yet") and re-runnable. A failure to store, or bytes // that are not an image at all, is an error — there is nothing to come back to. func (v *visionIntake) describe(ctx context.Context, req ipc.DescribeImageReq) (ipc.DescribeImageResp, error) { if len(req.Data) == 0 && req.ID == "" { return ipc.DescribeImageResp{}, fmt.Errorf("describe image: neither data nor id") } if len(req.Data) > 0 && req.ID != "" { // The contract says exactly one. Taking the ID branch and dropping the // bytes silently is the worst of the three possible answers: the caller // believes it sent a new image and nothing says otherwise. return ipc.DescribeImageResp{}, fmt.Errorf("describe image: both data and id given, send one") } var ( res vision.Result err error ) if req.ID != "" { res, err = v.in.Rerun(ctx, req.ID, req.Question) } else { res, err = v.in.Accept(ctx, req.Data, sourceOrDefault(req.Source), req.Question) } if res.Blob.ID == "" { // Nothing was stored: bad format, over the size cap, unwritable dir. return ipc.DescribeImageResp{}, fmt.Errorf("describe image: %w", err) } resp := ipc.DescribeImageResp{ ID: res.Blob.ID, Description: res.Description, Width: res.Image.Width, Height: res.Image.Height, } if err != nil { // Bytes are safe, words are not available. The log names the blob and the // reason; it never names what was in the picture. if errors.Is(err, vision.ErrDisabled) { log.Printf("vision: stored %s, no vision model configured", res.Blob) } else { log.Printf("vision: stored %s, describe failed: %v", res.Blob, err) } return resp, nil } if req.SaveNote { id, werr := v.writeNote(ctx, res) if werr != nil { // The description is still returned: losing the note is worse as a // silent failure than as a log line next to a successful answer. log.Printf("vision: note write for %s failed: %v", res.Blob, werr) } else { resp.NoteID = id } } log.Printf("vision: described %s (%dx%d)", res.Blob, res.Image.Width, res.Image.Height) return resp, nil } // noteMarker prefixes a stored description. Without it the note reads exactly // like something he told her, and it is not: it is a small VLM's guess about a // picture, embedded and recalled as if it were his own words. Four characters // of provenance in the text are cheaper than believing it later. const noteMarker = "Со снимка: " // writeNote stores the description as an ordinary note so it is recallable. The // note carries the blob id in its source, which is the only link back to the // bytes — the note text is words about the picture, never the picture. func (v *visionIntake) writeNote(ctx context.Context, res vision.Result) (int64, error) { var vec []float32 if v.emb != nil { // EmbedPassage, not Embed: a description is text being searched FOR, and // the e5 embedder is asymmetric. Backwards here makes it unfindable by // the question that should have matched it. var err error vec, err = router.EmbedPassage(ctx, v.emb, res.Description) if err != nil { return 0, fmt.Errorf("embed: %w", err) } } source := "media:image:" + res.Blob.ID[:12] return v.st.WriteNote(ctx, v.now(), noteMarker+res.Description, vec, source) } // sourceOrDefault labels a blob whose sender did not say where it came from. func sourceOrDefault(s string) string { if s == "" { return "unknown" } return s } // wireVision installs the IPC hook and starts the retention loop, or leaves the // hook nil so ipc.MethodDescribeImage reports ErrUnknownMethod. Called on both // startup paths (unlocked boot and passkey unlock) so vision behaves the same // either way. // // Returns the media keeper so the meeting recorder can share it: one blob store // with one retention loop holds both the images and the audio, which is the // whole point of internal/media being a shared package. nil ⇒ no media block, // and neither capability exists. func wireVision(ctx context.Context, wg *sync.WaitGroup, srv *ipc.Server, st *store.Store, emb router.Embedder, cfg *config.Config) *mediaKeeper { keeper := openMediaStore(cfg) if keeper == nil { return nil } // In the daemon's WaitGroup like every other loop in run: a prune deletes // files, and shutting down in the middle of one was the single loop nobody // waited for. wg.Add(1) go func() { defer wg.Done() keeper.runPrune(ctx) }() vi := newVisionIntake(keeper, st, emb, cfg) if vi == nil { return keeper } srv.DescribeImageFn = vi.describe return keeper }