vision: scope the note, settle the contract, wait for the prune
Saving a description writes recall corpus. writeNote embeds it under media:image:<id>, a source no enrollment owns, and the method sits at AuthRead, so any enrolled module could put a small VLM's guess into what Maven knows and have it come back in a later turn as something she believes. The describing half stays a read; save_note is now held to the same source-scope rule WriteFact is, and the stored text carries a marker saying it came off a picture. Three doc comments said the method exists only when vision is enabled and the code says otherwise. The code is right, and storing without describing is the state this box is in, so the comments were corrected rather than the behaviour. A request carrying both data and id used to take the id branch and drop the bytes without a word; it is refused. A media dir that cannot be created and a vision endpoint that is a typo were logged at wiring time and the capability just stayed off, which is the hardest kind of misconfiguration to notice. Both fail at startup. runPrune was the one loop started with a bare go and not in the daemon's WaitGroup, so shutdown did not wait for a prune that was deleting files. Found in review of #72.
This commit is contained in:
+7
-3
@@ -368,6 +368,11 @@ func run(args []string) error {
|
||||
|
||||
srv.StepUp = func(ctx context.Context) error { return passkeySess.Assert(ctx, auth.Scope{}) }
|
||||
|
||||
// wg is declared here rather than next to srv.Serve because the media
|
||||
// retention loop starts on this path too, and shutdown has to wait for a
|
||||
// prune in flight: it deletes files.
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Mail ingestion (Vikunja #246): the hook stays nil unless an email block is
|
||||
// configured and there is a llama-server to extract with, in which case
|
||||
// ipc.MethodIngestMail reports ErrUnknownMethod.
|
||||
@@ -376,7 +381,7 @@ func run(args []string) error {
|
||||
wireModelSwap(srv, phr, cfg)
|
||||
// Vision + the media blob store (Vikunja #252). Both stay dark without a
|
||||
// media block; MethodDescribeImage answers ErrUnknownMethod then.
|
||||
keeper := wireVision(ctx, srv, st, embedderOf(voiceW), cfg)
|
||||
keeper := wireVision(ctx, &wg, srv, st, embedderOf(voiceW), cfg)
|
||||
// The meeting recorder (Vikunja #253) shares that blob store and its
|
||||
// retention loop. Off unless a capture block enables it, in which case
|
||||
// all four capture methods answer ErrUnknownMethod.
|
||||
@@ -555,7 +560,7 @@ func run(args []string) error {
|
||||
srv.Check = (&auth.Gate{Enrollment: auth.NewFloorEnrollment(), Session: passkeySess}).Check
|
||||
wireMailIntake(srv, st, phr, cfg, evBus)
|
||||
wireModelSwap(srv, phr, cfg)
|
||||
keeper := wireVision(ctx, srv, st, embedderOf(voiceW), cfg)
|
||||
keeper := wireVision(ctx, &wg, srv, st, embedderOf(voiceW), cfg)
|
||||
wireCapture(srv, keeper, st, voiceW, phr, cfg)
|
||||
// Voice identification (Vikunja #255). Enrolment plumbing only until a
|
||||
// speaker-embedding model exists on disk; off entirely without a speaker
|
||||
@@ -622,7 +627,6 @@ func run(args []string) error {
|
||||
}
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
+35
-10
@@ -7,11 +7,14 @@
|
||||
// media.dir, prepares a downscaled JPEG, and asks a local vision server what it
|
||||
// is. The description comes back as words; nothing about the image is echoed.
|
||||
//
|
||||
// Off unless configured twice over: no `media` block ⇒ nowhere to keep the
|
||||
// bytes, so the method does not exist; no `vision` block with enabled + a local
|
||||
// endpoint ⇒ the store is wired but the describing half refuses, and the method
|
||||
// still does not exist. A surface cannot make Maven look at pictures by merely
|
||||
// sending one.
|
||||
// Off unless configured: no `media` block ⇒ nowhere to keep the bytes, so the
|
||||
// method does not exist and a surface cannot make Maven accept a photo by
|
||||
// merely sending one. A `media` block with no `vision` block is a real state,
|
||||
// the one this box is in today: the store is wired, the method exists, the
|
||||
// bytes are kept and the reply says she cannot read the picture yet. That reply
|
||||
// is re-runnable by id on the day a vision model lands, which is the reason to
|
||||
// keep the bytes at all. Saving the description as a note needs more than the
|
||||
// read rung — see the scope check on auth.ImageNoteSource.
|
||||
//
|
||||
// Two things this file deliberately does not do:
|
||||
//
|
||||
@@ -29,6 +32,7 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
@@ -64,12 +68,14 @@ func openMediaStore(cfg *config.Config) *mediaKeeper {
|
||||
if !filepath.IsAbs(dir) && cfg.StateDir != "" {
|
||||
dir = filepath.Join(cfg.StateDir, dir)
|
||||
}
|
||||
st, err := media.Open(dir, cfg.Media.MaxBytes, time.Duration(cfg.Media.Retention))
|
||||
st, err := media.OpenWithBudget(dir, cfg.Media.MaxBytes, cfg.Media.MaxTotalBytes,
|
||||
time.Duration(cfg.Media.Retention))
|
||||
if err != nil {
|
||||
log.Printf("media: %v — image and audio intake disabled", err)
|
||||
return nil
|
||||
}
|
||||
log.Printf("media: blob store at %s, retention %s", st.Dir(), st.Retention())
|
||||
log.Printf("media: blob store at %s, retention %s, %d of %d bytes used",
|
||||
st.Dir(), st.Retention(), st.Total(), st.Budget())
|
||||
return &mediaKeeper{store: st}
|
||||
}
|
||||
|
||||
@@ -159,6 +165,12 @@ func (v *visionIntake) describe(ctx context.Context, req ipc.DescribeImageReq) (
|
||||
if len(req.Data) == 0 && req.ID == "" {
|
||||
return ipc.DescribeImageResp{}, fmt.Errorf("describe image: neither data nor id")
|
||||
}
|
||||
if len(req.Data) > 0 && req.ID != "" {
|
||||
// The contract says exactly one. Taking the ID branch and dropping the
|
||||
// bytes silently is the worst of the three possible answers: the caller
|
||||
// believes it sent a new image and nothing says otherwise.
|
||||
return ipc.DescribeImageResp{}, fmt.Errorf("describe image: both data and id given, send one")
|
||||
}
|
||||
|
||||
var (
|
||||
res vision.Result
|
||||
@@ -205,6 +217,12 @@ func (v *visionIntake) describe(ctx context.Context, req ipc.DescribeImageReq) (
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// noteMarker prefixes a stored description. Without it the note reads exactly
|
||||
// like something he told her, and it is not: it is a small VLM's guess about a
|
||||
// picture, embedded and recalled as if it were his own words. Four characters
|
||||
// of provenance in the text are cheaper than believing it later.
|
||||
const noteMarker = "Со снимка: "
|
||||
|
||||
// writeNote stores the description as an ordinary note so it is recallable. The
|
||||
// note carries the blob id in its source, which is the only link back to the
|
||||
// bytes — the note text is words about the picture, never the picture.
|
||||
@@ -221,7 +239,7 @@ func (v *visionIntake) writeNote(ctx context.Context, res vision.Result) (int64,
|
||||
}
|
||||
}
|
||||
source := "media:image:" + res.Blob.ID[:12]
|
||||
return v.st.WriteNote(ctx, v.now(), res.Description, vec, source)
|
||||
return v.st.WriteNote(ctx, v.now(), noteMarker+res.Description, vec, source)
|
||||
}
|
||||
|
||||
// sourceOrDefault labels a blob whose sender did not say where it came from.
|
||||
@@ -241,12 +259,19 @@ func sourceOrDefault(s string) string {
|
||||
// with one retention loop holds both the images and the audio, which is the
|
||||
// whole point of internal/media being a shared package. nil ⇒ no media block,
|
||||
// and neither capability exists.
|
||||
func wireVision(ctx context.Context, srv *ipc.Server, st *store.Store, emb router.Embedder, cfg *config.Config) *mediaKeeper {
|
||||
func wireVision(ctx context.Context, wg *sync.WaitGroup, srv *ipc.Server, st *store.Store, emb router.Embedder, cfg *config.Config) *mediaKeeper {
|
||||
keeper := openMediaStore(cfg)
|
||||
if keeper == nil {
|
||||
return nil
|
||||
}
|
||||
go keeper.runPrune(ctx)
|
||||
// In the daemon's WaitGroup like every other loop in run: a prune deletes
|
||||
// files, and shutting down in the middle of one was the single loop nobody
|
||||
// waited for.
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
keeper.runPrune(ctx)
|
||||
}()
|
||||
|
||||
vi := newVisionIntake(keeper, st, emb, cfg)
|
||||
if vi == nil {
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"image"
|
||||
"image/png"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/config"
|
||||
"github.com/kami/maven/internal/ipc"
|
||||
"github.com/kami/maven/internal/media"
|
||||
"github.com/kami/maven/internal/vision"
|
||||
)
|
||||
|
||||
func testIntake(t *testing.T) *visionIntake {
|
||||
t.Helper()
|
||||
st := newTestStore(t)
|
||||
blobs, err := media.Open(t.TempDir(), 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &visionIntake{
|
||||
in: vision.NewIntake(blobs, vision.Disabled{}, 0),
|
||||
st: st,
|
||||
now: time.Now,
|
||||
}
|
||||
}
|
||||
|
||||
// The contract says exactly one of Data or ID. Taking the ID branch and
|
||||
// dropping the bytes silently is the worst of the three possible answers: the
|
||||
// caller believes it sent a new image and nothing says otherwise.
|
||||
func TestDescribeRefusesBothDataAndID(t *testing.T) {
|
||||
v := testIntake(t)
|
||||
_, err := v.describe(context.Background(), ipc.DescribeImageReq{
|
||||
Data: []byte("bytes"), ID: strings.Repeat("a", 64),
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("both data and id must be refused")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "send one") {
|
||||
t.Fatalf("err = %v, want it to name the contract", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Vision being off does not remove the method: the bytes are stored and the
|
||||
// answer says she cannot read the picture yet, which is re-runnable by id. That
|
||||
// is the state this box is in today, and three doc comments used to claim the
|
||||
// opposite.
|
||||
func TestVisionOffStillStores(t *testing.T) {
|
||||
v := testIntake(t)
|
||||
var buf bytes.Buffer
|
||||
if err := png.Encode(&buf, image.NewRGBA(image.Rect(0, 0, 4, 4))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, err := v.describe(context.Background(), ipc.DescribeImageReq{Data: buf.Bytes(), Source: "web:upload"})
|
||||
if err != nil {
|
||||
t.Fatalf("storing must succeed even with no vision model: %v", err)
|
||||
}
|
||||
if len(resp.ID) != 64 {
|
||||
t.Fatalf("no blob id came back: %+v", resp)
|
||||
}
|
||||
if resp.Description != "" {
|
||||
t.Errorf("description = %q, want none", resp.Description)
|
||||
}
|
||||
// And with no media block at all the method does not exist.
|
||||
if vi := newVisionIntake(nil, nil, nil, &config.Config{}); vi != nil {
|
||||
t.Fatal("no media block must leave the method nonexistent")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user