Store and describe images through a shared media intake (#252)
Vision needs a second model this box does not have, so the shipped half is the part that works without one: an image arrives, is sniffed, is stored content-addressed, and is prepared for inference. The describing half is written and tested against a fake server, and refuses any endpoint that is not on this box. internal/media is the intake all three senses share — hearing and speaker recognition store their audio in the same place under the same retention. Blobs stay out of the sqlite store; only the derived text becomes a note, and only when the caller asks. Retention is enforced by an hourly prune loop rather than by a comment. The plan's RemoteProvider step is refused: no cloud model, inference stays on the box, and vision.NewLocal validates that at construction.
This commit is contained in:
@@ -18,6 +18,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/delivery/ntfysink"
|
||||
@@ -193,6 +194,17 @@ type Config struct {
|
||||
// nil ⇒ no capability-aware routing.
|
||||
Hexis *HexisConfig `json:"hexis,omitempty"`
|
||||
|
||||
// Vision — image understanding (Vikunja #252). nil / absent ⇒ she cannot
|
||||
// look at pictures at all: the intake refuses, and no vision server is
|
||||
// contacted. See VisionConfig.
|
||||
Vision *VisionConfig `json:"vision,omitempty"`
|
||||
|
||||
// Media — where images and captured audio are kept on disk, and for how
|
||||
// long. nil / absent ⇒ no blob store is wired, which is what disables both
|
||||
// vision intake and meeting capture regardless of their own blocks: nothing
|
||||
// in this repo holds a recording only in memory. See MediaConfig.
|
||||
Media *MediaConfig `json:"media,omitempty"`
|
||||
|
||||
// MCP — Model Context Protocol servers Maven connects OUT to (Vikunja
|
||||
// #251). nil / absent / no enabled server ⇒ no connection is made and no
|
||||
// tool is discovered, like every other capability that reaches outside the
|
||||
@@ -474,6 +486,78 @@ type VoiceConfig struct {
|
||||
ToolTimeout Duration `json:"tool_timeout,omitempty"`
|
||||
}
|
||||
|
||||
// MediaConfig — the on-disk blob store for images and captured audio
|
||||
// (internal/media). It is shared by all three senses: vision intake, meeting
|
||||
// capture, and speaker enrolment samples all write here.
|
||||
//
|
||||
// Absent ⇒ off, and off means Maven cannot accept an image or start a recording
|
||||
// at all. That default is deliberate: a capability that keeps photos and audio of
|
||||
// people on disk should require someone to have typed a path.
|
||||
type MediaConfig struct {
|
||||
// Dir — the blob store root, created 0700. Relative paths resolve against
|
||||
// StateDir. Required; an empty dir means the store is not wired.
|
||||
Dir string `json:"dir,omitempty"`
|
||||
|
||||
// Retention — how long a blob is kept before the tick prunes it. 0 ⇒
|
||||
// media.DefaultRetention (7 days). This is the knob that stops recordings
|
||||
// of people accumulating; raising it past a few weeks should need a reason.
|
||||
Retention Duration `json:"retention,omitempty"`
|
||||
|
||||
// MaxBytes — per-blob cap. 0 ⇒ media.DefaultMaxBytes (64 MiB).
|
||||
MaxBytes int64 `json:"max_bytes,omitempty"`
|
||||
}
|
||||
|
||||
// StoreDir reports the configured blob directory, or "" when media is not
|
||||
// wired. Safe on a nil receiver.
|
||||
func (m *MediaConfig) StoreDir() string {
|
||||
if m == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(m.Dir)
|
||||
}
|
||||
|
||||
// VisionConfig — the vision provider (internal/vision, docs/plans/07-vision.md).
|
||||
//
|
||||
// Absent, or enabled=false, ⇒ the daemon wires vision.Disabled and every attempt
|
||||
// to look at an image answers that vision is not set up. There is no cloud
|
||||
// option in this block on purpose: Endpoint must be a loopback or private
|
||||
// address and internal/vision refuses anything else at startup, because
|
||||
// inference stays on the box and a photo of his flat is the last thing to make
|
||||
// an exception for.
|
||||
type VisionConfig struct {
|
||||
// Enabled — may she look at images. Default false.
|
||||
Enabled bool `json:"enabled,omitempty"`
|
||||
|
||||
// Endpoint — base URL of a llama-server running a vision model with its
|
||||
// mmproj, e.g. "http://127.0.0.1:8081". Loopback / private only.
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
|
||||
// Model — model name sent in the request. llama-server ignores it.
|
||||
Model string `json:"model,omitempty"`
|
||||
|
||||
// MaxDim — longest edge the image is scaled to before inference. 0 ⇒
|
||||
// media.DefaultMaxDim (896).
|
||||
MaxDim int `json:"max_dim,omitempty"`
|
||||
|
||||
// MaxTokens — cap on the description. 0 ⇒ vision.DefaultMaxTokens (300).
|
||||
MaxTokens int `json:"max_tokens,omitempty"`
|
||||
|
||||
// Timeout — per-description budget. 0 ⇒ vision.DefaultTimeout (90s). A small
|
||||
// VLM on an iGPU is slow; a tight timeout here just means no answer ever.
|
||||
Timeout Duration `json:"timeout,omitempty"`
|
||||
|
||||
// Prompt — the default question when he only sent a picture. Empty ⇒
|
||||
// vision.DefaultPrompt (Russian, "опиши что на изображении").
|
||||
Prompt string `json:"prompt,omitempty"`
|
||||
}
|
||||
|
||||
// LooksAtImages reports whether vision is configured well enough to try. Safe on
|
||||
// a nil receiver, and false without an endpoint — enabled with nothing to talk
|
||||
// to is a misconfiguration, not a capability.
|
||||
func (v *VisionConfig) LooksAtImages() bool {
|
||||
return v != nil && v.Enabled && strings.TrimSpace(v.Endpoint) != ""
|
||||
}
|
||||
|
||||
// WeatherConfig configures the weather provider for voice queries.
|
||||
type WeatherConfig struct {
|
||||
Provider string `json:"provider,omitempty"` // "open-meteo" or "" → stub
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Absent blocks must read as off on a nil receiver: the daemon calls these
|
||||
// helpers before it knows whether the operator configured anything.
|
||||
func TestSensesOffByDefault(t *testing.T) {
|
||||
var cfg Config
|
||||
if cfg.Media.StoreDir() != "" {
|
||||
t.Error("media store dir is set with no media block")
|
||||
}
|
||||
if cfg.Vision.LooksAtImages() {
|
||||
t.Error("vision is on with no vision block")
|
||||
}
|
||||
}
|
||||
|
||||
// enabled with nothing to talk to is a misconfiguration, not a capability.
|
||||
func TestVisionNeedsBothEnabledAndEndpoint(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
v *VisionConfig
|
||||
want bool
|
||||
}{
|
||||
{"absent", nil, false},
|
||||
{"endpoint but not enabled", &VisionConfig{Endpoint: "http://127.0.0.1:8081"}, false},
|
||||
{"enabled but no endpoint", &VisionConfig{Enabled: true}, false},
|
||||
{"enabled, blank endpoint", &VisionConfig{Enabled: true, Endpoint: " "}, false},
|
||||
{"both", &VisionConfig{Enabled: true, Endpoint: "http://127.0.0.1:8081"}, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := c.v.LooksAtImages(); got != c.want {
|
||||
t.Errorf("%s: LooksAtImages() = %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSensesBlocksParseFromJSON(t *testing.T) {
|
||||
raw := `{
|
||||
"db_path": "/tmp/x.db",
|
||||
"socket_path": "/tmp/x.sock",
|
||||
"media": {"dir": "media", "retention": "48h", "max_bytes": 1048576},
|
||||
"vision": {
|
||||
"enabled": true,
|
||||
"endpoint": "http://127.0.0.1:8081",
|
||||
"model": "qwen2.5-vl",
|
||||
"max_dim": 640,
|
||||
"max_tokens": 200,
|
||||
"timeout": "45s",
|
||||
"prompt": "Что тут?"
|
||||
}
|
||||
}`
|
||||
var cfg Config
|
||||
if err := json.Unmarshal([]byte(raw), &cfg); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
if cfg.Media.StoreDir() != "media" {
|
||||
t.Errorf("media dir = %q", cfg.Media.StoreDir())
|
||||
}
|
||||
if time.Duration(cfg.Media.Retention) != 48*time.Hour {
|
||||
t.Errorf("retention = %v", time.Duration(cfg.Media.Retention))
|
||||
}
|
||||
if cfg.Media.MaxBytes != 1<<20 {
|
||||
t.Errorf("max_bytes = %d", cfg.Media.MaxBytes)
|
||||
}
|
||||
if !cfg.Vision.LooksAtImages() {
|
||||
t.Fatal("vision did not parse as enabled")
|
||||
}
|
||||
if cfg.Vision.MaxDim != 640 || cfg.Vision.MaxTokens != 200 {
|
||||
t.Errorf("vision limits = %+v", cfg.Vision)
|
||||
}
|
||||
if time.Duration(cfg.Vision.Timeout) != 45*time.Second {
|
||||
t.Errorf("vision timeout = %v", time.Duration(cfg.Vision.Timeout))
|
||||
}
|
||||
if cfg.Vision.Prompt != "Что тут?" {
|
||||
t.Errorf("prompt = %q", cfg.Vision.Prompt)
|
||||
}
|
||||
}
|
||||
|
||||
// A media dir set with no vision block is a valid state, and the useful one on a
|
||||
// box with no vision model: images can be kept, they just cannot be described.
|
||||
func TestMediaWithoutVisionIsValid(t *testing.T) {
|
||||
var cfg Config
|
||||
if err := json.Unmarshal([]byte(`{"media":{"dir":"/srv/media"}}`), &cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.Media.StoreDir() != "/srv/media" {
|
||||
t.Errorf("dir = %q", cfg.Media.StoreDir())
|
||||
}
|
||||
if cfg.Vision.LooksAtImages() {
|
||||
t.Error("vision came on by itself")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user