Maven/cmd/mavend/vision.go

// mavend/vision.go — core's half of image understanding (Vikunja #252,
// docs/plans/07-vision.md).
//
// The split: any surface that can receive a picture (mavweb upload, a Telegram
// photo through mavpoll, a path he names) hands the bytes to core over
// ipc.MethodDescribeImage. Core stores them content-addressed under
// media.dir, prepares a downscaled JPEG, and asks a local vision server what it
// is. The description comes back as words; nothing about the image is echoed.
//
// Off unless configured twice over: no `media` block ⇒ nowhere to keep the
// bytes, so the method does not exist; no `vision` block with enabled + a local
// endpoint ⇒ the store is wired but the describing half refuses, and the method
// still does not exist. A surface cannot make Maven look at pictures by merely
// sending one.
//
// Two things this file deliberately does not do:
//
//   - No cloud vision call, ever. internal/vision refuses a non-private
//     endpoint at construction; there is no config shape here that could reach
//     an upstream API even if someone wanted one.
//   - No automatic memory. SaveNote is opt-in per call. Glancing at a screenshot
//     is not the same act as remembering it, and a 1.7B-class VLM's guess about
//     a photo is not a fact worth carrying around.
package main

import (
	"context"
	"errors"
	"fmt"
	"log"
	"path/filepath"
	"time"

	"github.com/kami/maven/internal/config"
	"github.com/kami/maven/internal/ipc"
	"github.com/kami/maven/internal/media"
	"github.com/kami/maven/internal/router"
	"github.com/kami/maven/internal/store"
	"github.com/kami/maven/internal/vision"
)

// prunePeriod — how often stored blobs are checked against media.retention.
// Hourly is far more often than needed for a 7-day retention and costs a
// directory walk over a handful of sidecars; the point is that the promise is
// kept by a loop that runs, not by an operator remembering a cron.
const prunePeriod = time.Hour

// mediaKeeper — the blob store plus the loop that enforces its retention. The
// two are one object because a store without the loop is a directory that grows
// forever, and shipping that would break the only interesting promise this
// capability makes.
type mediaKeeper struct {
	store *media.Store
}

// openMediaStore builds the blob store from config, or returns nil when media is
// not configured. A relative dir resolves against StateDir, the same rule the db
// and socket paths follow.
func openMediaStore(cfg *config.Config) *mediaKeeper {
	dir := cfg.Media.StoreDir()
	if dir == "" {
		return nil
	}
	if !filepath.IsAbs(dir) && cfg.StateDir != "" {
		dir = filepath.Join(cfg.StateDir, dir)
	}
	st, err := media.Open(dir, cfg.Media.MaxBytes, time.Duration(cfg.Media.Retention))
	if err != nil {
		log.Printf("media: %v — image and audio intake disabled", err)
		return nil
	}
	log.Printf("media: blob store at %s, retention %s", st.Dir(), st.Retention())
	return &mediaKeeper{store: st}
}

// runPrune deletes over-retention blobs on a loop until ctx ends. It prunes once
// immediately, so a daemon restarted after a long downtime does not sit on a
// month of stale recordings until the first tick.
func (k *mediaKeeper) runPrune(ctx context.Context) {
	prune := func() {
		n, err := k.store.Prune()
		if err != nil {
			log.Printf("media: prune: %v", err)
			return
		}
		if n > 0 {
			log.Printf("media: pruned %d blob(s) older than %s", n, k.store.Retention())
		}
	}
	prune()
	t := time.NewTicker(prunePeriod)
	defer t.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-t.C:
			prune()
		}
	}
}

// visionIntake — one image at a time: store, prepare, describe, optionally note.
type visionIntake struct {
	in  *vision.Intake
	st  *store.Store
	emb router.Embedder
	now func() time.Time
}

// newVisionIntake returns nil when there is nothing to wire. keeper == nil means
// no media block, which disables the method outright; a missing or disabled
// vision block still wires the method, because storing an image and answering
// "I can't look at it yet" is more useful than pretending the surface does not
// exist — and it is exactly the state this box is in until a vision model is on
// disk.
func newVisionIntake(keeper *mediaKeeper, st *store.Store, emb router.Embedder, cfg *config.Config) *visionIntake {
	if keeper == nil {
		return nil
	}
	vc := cfg.Vision
	maxDim := 0
	var provider vision.Provider = vision.Disabled{}
	if vc.LooksAtImages() {
		p, err := vision.NewLocal(vision.Config{
			Endpoint:  vc.Endpoint,
			Model:     vc.Model,
			Timeout:   time.Duration(vc.Timeout),
			MaxTokens: vc.MaxTokens,
			Prompt:    vc.Prompt,
		})
		if err != nil {
			// A public endpoint, a hostname, a bad URL. Logged once here rather
			// than failing every turn, and the store still works.
			log.Printf("vision: %v — she can store images but not describe them", err)
		} else {
			provider = p
			maxDim = vc.MaxDim
			log.Printf("vision: enabled against %s", p.Endpoint())
		}
	} else {
		log.Printf("vision: not configured — images are stored, not described")
	}
	return &visionIntake{
		in:  vision.NewIntake(keeper.store, provider, maxDim),
		st:  st,
		emb: emb,
		now: time.Now,
	}
}

// describe handles one ipc.MethodDescribeImage call.
//
// A description failure is NOT an error out of this method when the bytes were
// stored: the caller gets the id and an empty description, which is honest ("it
// is kept, I cannot read it yet") and re-runnable. A failure to store, or bytes
// that are not an image at all, is an error — there is nothing to come back to.
func (v *visionIntake) describe(ctx context.Context, req ipc.DescribeImageReq) (ipc.DescribeImageResp, error) {
	if len(req.Data) == 0 && req.ID == "" {
		return ipc.DescribeImageResp{}, fmt.Errorf("describe image: neither data nor id")
	}

	var (
		res vision.Result
		err error
	)
	if req.ID != "" {
		res, err = v.in.Rerun(ctx, req.ID, req.Question)
	} else {
		res, err = v.in.Accept(ctx, req.Data, sourceOrDefault(req.Source), req.Question)
	}
	if res.Blob.ID == "" {
		// Nothing was stored: bad format, over the size cap, unwritable dir.
		return ipc.DescribeImageResp{}, fmt.Errorf("describe image: %w", err)
	}

	resp := ipc.DescribeImageResp{
		ID:          res.Blob.ID,
		Description: res.Description,
		Width:       res.Image.Width,
		Height:      res.Image.Height,
	}
	if err != nil {
		// Bytes are safe, words are not available. The log names the blob and the
		// reason; it never names what was in the picture.
		if errors.Is(err, vision.ErrDisabled) {
			log.Printf("vision: stored %s, no vision model configured", res.Blob)
		} else {
			log.Printf("vision: stored %s, describe failed: %v", res.Blob, err)
		}
		return resp, nil
	}

	if req.SaveNote {
		id, werr := v.writeNote(ctx, res)
		if werr != nil {
			// The description is still returned: losing the note is worse as a
			// silent failure than as a log line next to a successful answer.
			log.Printf("vision: note write for %s failed: %v", res.Blob, werr)
		} else {
			resp.NoteID = id
		}
	}
	log.Printf("vision: described %s (%dx%d)", res.Blob, res.Image.Width, res.Image.Height)
	return resp, nil
}

// writeNote stores the description as an ordinary note so it is recallable. The
// note carries the blob id in its source, which is the only link back to the
// bytes — the note text is words about the picture, never the picture.
func (v *visionIntake) writeNote(ctx context.Context, res vision.Result) (int64, error) {
	var vec []float32
	if v.emb != nil {
		// EmbedPassage, not Embed: a description is text being searched FOR, and
		// the e5 embedder is asymmetric. Backwards here makes it unfindable by
		// the question that should have matched it.
		var err error
		vec, err = router.EmbedPassage(ctx, v.emb, res.Description)
		if err != nil {
			return 0, fmt.Errorf("embed: %w", err)
		}
	}
	source := "media:image:" + res.Blob.ID[:12]
	return v.st.WriteNote(ctx, v.now(), res.Description, vec, source)
}

// sourceOrDefault labels a blob whose sender did not say where it came from.
func sourceOrDefault(s string) string {
	if s == "" {
		return "unknown"
	}
	return s
}

// wireVision installs the IPC hook and starts the retention loop, or leaves the
// hook nil so ipc.MethodDescribeImage reports ErrUnknownMethod. Called on both
// startup paths (unlocked boot and passkey unlock) so vision behaves the same
// either way.
//
// Returns the media keeper so the meeting recorder can share it: one blob store
// with one retention loop holds both the images and the audio, which is the
// whole point of internal/media being a shared package. nil ⇒ no media block,
// and neither capability exists.
func wireVision(ctx context.Context, srv *ipc.Server, st *store.Store, emb router.Embedder, cfg *config.Config) *mediaKeeper {
	keeper := openMediaStore(cfg)
	if keeper == nil {
		return nil
	}
	go keeper.runPrune(ctx)

	vi := newVisionIntake(keeper, st, emb, cfg)
	if vi == nil {
		return keeper
	}
	srv.DescribeImageFn = vi.describe
	return keeper
}