Store and describe images through a shared media intake (#252)
Vision needs a second model this box does not have, so the shipped half is the part that works without one: an image arrives, is sniffed, is stored content-addressed, and is prepared for inference. The describing half is written and tested against a fake server, and refuses any endpoint that is not on this box. internal/media is the intake all three senses share — hearing and speaker recognition store their audio in the same place under the same retention. Blobs stay out of the sqlite store; only the derived text becomes a note, and only when the caller asks. Retention is enforced by an hourly prune loop rather than by a comment. The plan's RemoteProvider step is refused: no cloud model, inference stays on the box, and vision.NewLocal validates that at construction.
This commit is contained in:
@@ -176,6 +176,51 @@ type IngestMailResp struct {
|
||||
Skipped bool `json:"skipped,omitempty"`
|
||||
}
|
||||
|
||||
// DescribeImageReq — one image handed to core to look at (Vikunja #252).
|
||||
//
|
||||
// Data is the raw image file as received (png / jpeg / gif). Core sniffs it and
|
||||
// refuses anything else; a declared content type is not part of this request
|
||||
// because the sender's claim about its own bytes is not evidence. Base64 on the
|
||||
// wire via the usual JSON marshal of []byte.
|
||||
//
|
||||
// Question is what he asked about the picture ("что тут написано?"). Empty ⇒
|
||||
// core uses its configured default prompt.
|
||||
//
|
||||
// Source is provenance recorded on the stored blob: "telegram", "web:upload".
|
||||
//
|
||||
// Exactly one of Data or ID is set. ID re-describes an image core already has —
|
||||
// a different question, or the first attempt that succeeds after a vision model
|
||||
// finally lands on disk.
|
||||
//
|
||||
// The method exists only when core has both a media store and an enabled vision
|
||||
// block; otherwise it answers ErrUnknownMethod, which is what "off unless
|
||||
// configured" looks like at the wire. A surface cannot make Maven look at
|
||||
// pictures by merely sending one.
|
||||
type DescribeImageReq struct {
|
||||
Data []byte `json:"data,omitempty"`
|
||||
ID string `json:"id,omitempty"`
|
||||
Source string `json:"source,omitempty"`
|
||||
Question string `json:"question,omitempty"`
|
||||
// SaveNote — also write the description as a note (source
|
||||
// "media:image:<id-prefix>") so it is recallable later. Default false: a
|
||||
// glance at a screenshot is not automatically a memory.
|
||||
SaveNote bool `json:"save_note,omitempty"`
|
||||
}
|
||||
|
||||
// DescribeImageResp — what she saw. ID is the stored blob's content address, and
|
||||
// it is set even when Description is empty because the description failed: the
|
||||
// bytes are on disk and the same id can be retried. NoteID is non-zero only when
|
||||
// SaveNote was set and the write succeeded.
|
||||
//
|
||||
// The image itself is never echoed back.
|
||||
type DescribeImageResp struct {
|
||||
ID string `json:"id"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Width int `json:"width,omitempty"`
|
||||
Height int `json:"height,omitempty"`
|
||||
NoteID int64 `json:"note_id,omitempty"`
|
||||
}
|
||||
|
||||
// SwapModelReq — load another resident model without restarting the daemon
|
||||
// (Vikunja #250). ModelPath must be one of the paths in phraser.swap_models;
|
||||
// anything else is ErrForbidden, and an unconfigured allowlist makes the whole
|
||||
|
||||
@@ -460,6 +460,19 @@ func (c *Client) IngestMail(ctx context.Context, req IngestMailReq) (IngestMailR
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// DescribeImage hands one image to core to look at (Vikunja #252).
|
||||
// ErrUnknownMethod means core has no media store or vision is off — the caller
|
||||
// should stop asking, not retry. A response with an ID and an empty Description
|
||||
// means the bytes were stored but nothing could describe them yet, which is the
|
||||
// expected state on a box with no vision model on disk.
|
||||
func (c *Client) DescribeImage(ctx context.Context, req DescribeImageReq) (DescribeImageResp, error) {
|
||||
var r DescribeImageResp
|
||||
if err := c.call(ctx, MethodDescribeImage, req, &r); err != nil {
|
||||
return DescribeImageResp{}, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// SwapModel asks core to load another resident model (Vikunja #250).
|
||||
// ErrUnknownMethod means core has no phraser.swap_models allowlist configured;
|
||||
// ErrForbidden means the path is not on it, or step-up was not asserted. A
|
||||
|
||||
+31
-4
@@ -449,6 +449,16 @@ type Server struct {
|
||||
SwapModelFn SwapModelFunc
|
||||
ModelStatusFn ModelStatusFunc
|
||||
|
||||
// DescribeImageFn — looks at one image (Vikunja #252). Set by the daemon only
|
||||
// when a media store is configured AND vision is enabled with a local
|
||||
// endpoint; nil ⇒ MethodDescribeImage answers ErrUnknownMethod, so a surface
|
||||
// cannot make Maven accept a photo by merely sending one.
|
||||
//
|
||||
// It bypasses CoreAPI for the same reason IngestMailFn does: it needs a blob
|
||||
// store and a vision server, neither of which is a store operation, and no
|
||||
// other CoreAPI implementation should have to carry it.
|
||||
DescribeImageFn DescribeImageFunc
|
||||
|
||||
// UnlockFn — unwraps the store encryption key from the wrapped blob using
|
||||
// the passkey credential public key, opens the encrypted store, and wires
|
||||
// the rest of the daemon (voice, loop, delivery). Set by the daemon when
|
||||
@@ -476,6 +486,9 @@ type ModelStatusFunc func(ctx context.Context) (ModelStatusResp, error)
|
||||
// IngestMailFunc — core-side mail extraction. Returns what was captured.
|
||||
type IngestMailFunc func(ctx context.Context, req IngestMailReq) (IngestMailResp, error)
|
||||
|
||||
// DescribeImageFunc — core-side image intake + description.
|
||||
type DescribeImageFunc func(ctx context.Context, req DescribeImageReq) (DescribeImageResp, error)
|
||||
|
||||
// CheckFunc — the auth hook signature. Wired by the daemon (auth.Gate.Check
|
||||
// satisfies this); dispatch calls it once per request after param-unmarshal
|
||||
// independence (it gets the raw params, may unmarshal what it needs — ipc
|
||||
@@ -644,10 +657,10 @@ func withoutParams[R any](fn func(ctx context.Context, api CoreAPI) (R, error))
|
||||
// is still honored on the very next request with no extra plumbing here.
|
||||
//
|
||||
// MethodAssertStepUp, MethodStoreEncryptionKey, MethodUnlock,
|
||||
// MethodIngestMail, MethodSwapModel and MethodModelStatus are NOT in this
|
||||
// table: they bypass CoreAPI entirely
|
||||
// (s.StepUp / s.WrapKeyFn / s.UnlockFn / s.IngestMailFn), so dispatch
|
||||
// special-cases them before consulting the table.
|
||||
// MethodIngestMail, MethodSwapModel, MethodModelStatus and
|
||||
// MethodDescribeImage are NOT in this table: they bypass CoreAPI entirely
|
||||
// (s.StepUp / s.WrapKeyFn / s.UnlockFn / s.IngestMailFn / s.DescribeImageFn),
|
||||
// so dispatch special-cases them before consulting the table.
|
||||
var methodTable = map[Method]handlerFunc{
|
||||
MethodWriteFact: withParams(func(ctx context.Context, api CoreAPI, p WriteFactReq) (idResp, error) {
|
||||
id, err := api.WriteFact(ctx, p)
|
||||
@@ -923,6 +936,20 @@ func (s *Server) dispatch(ctx context.Context, req Request) (json.RawMessage, er
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method)
|
||||
|
||||
case MethodDescribeImage:
|
||||
if s.DescribeImageFn != nil {
|
||||
var p DescribeImageReq
|
||||
if err := unmarshalParams(req.Params, &p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := s.DescribeImageFn(ctx, p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return marshalResult(resp), nil
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnknownMethod, req.Method)
|
||||
|
||||
case MethodModelStatus:
|
||||
if s.ModelStatusFn != nil {
|
||||
resp, err := s.ModelStatusFn(ctx)
|
||||
|
||||
@@ -54,6 +54,7 @@ const (
|
||||
MethodIngestMail Method = "ingest_mail"
|
||||
MethodSwapModel Method = "swap_model"
|
||||
MethodModelStatus Method = "model_status"
|
||||
MethodDescribeImage Method = "describe_image"
|
||||
)
|
||||
|
||||
// Request — one frame from module to core. Params is the JSON-encoded argument
|
||||
|
||||
Reference in New Issue
Block a user