Maven/internal/router/embedder.go

package router

import (
	"context"
	"fmt"
	"math"
	"unicode"
)

// Embedder produces a dense vector for a text utterance. The classifier is
// nearest-centroid over labeled-intent example vectors; the embedder is the
// one impure seam — the production path is the multilingual ONNX int8 model
// (multilingual-e5-small or paraphrase-multilingual-MiniLM-L12-v2, ~120mb,
// bilingual ru+en native, no separate ru/en path). That model is a later
// module; anything deterministic + dimension-fixed works here, which keeps
// the classifier + cascade unit-testable without the model loaded.
type Embedder interface {
	Dim() int
	Embed(ctx context.Context, text string) ([]float32, error)
	Close() error
}

// IdentifiedEmbedder — an embedder that can name itself. The name goes into
// the DB next to the vectors it wrote, so a later model swap is caught instead
// of silently returning nonsense scores (Vikunja #378).
type IdentifiedEmbedder interface {
	Embedder
	ID() string
}

// EmbedderID is the stable string stored alongside the vectors. It comes from
// the embedder itself — nobody hand-types a model name twice — and changes
// whenever the model or its dimension changes.
func EmbedderID(e Embedder) string {
	if i, ok := e.(IdentifiedEmbedder); ok {
		return i.ID()
	}
	return fmt.Sprintf("unknown@%d", e.Dim())
}

// AsymmetricEmbedder — an embedder that wants to know whether a text is a
// search query or a stored passage. Recall is asymmetric: a short question
// goes in, a longer note comes out. The e5 family is trained for exactly that
// and needs the side written into the text ("query: " / "passage: ").
//
// Optional on purpose: HashEmbedder has no such notion, so callers go through
// EmbedQuery and EmbedPassage below, which fall back to plain Embed.
type AsymmetricEmbedder interface {
	Embedder
	EmbedQuery(ctx context.Context, text string) ([]float32, error)
	EmbedPassage(ctx context.Context, text string) ([]float32, error)
}

// EmbedQuery embeds text that is being searched WITH — a question.
func EmbedQuery(ctx context.Context, e Embedder, text string) ([]float32, error) {
	if a, ok := e.(AsymmetricEmbedder); ok {
		return a.EmbedQuery(ctx, text)
	}
	return e.Embed(ctx, text)
}

// EmbedPassage embeds text that is being searched FOR — a note or a fact on
// its way into the store. Store and lookup must use these two calls, not one
// of them twice, or the asymmetry buys nothing.
func EmbedPassage(ctx context.Context, e Embedder, text string) ([]float32, error) {
	if a, ok := e.(AsymmetricEmbedder); ok {
		return a.EmbedPassage(ctx, text)
	}
	return e.Embed(ctx, text)
}

// HashEmbedder — a deterministic bag-of-words embedder used for tests and as a
// non-zero default floor. NOT semantically meaningful across languages; the
// real classifier swaps in the multilingual ONNX model wholesale.
//
// Token collisions are the point: same surface words ⇒ similar vectors ⇒ the
// centroid math is testable. Each token hashes into a dimension; weights
// accumulate then L2-normalize so cosine similarity is a clean inner product.
type HashEmbedder struct {
	dim int
}

func NewHashEmbedder(dim int) *HashEmbedder {
	if dim <= 0 {
		dim = 128
	}
	return &HashEmbedder{dim: dim}
}

func (h *HashEmbedder) Dim() int { return h.dim }

// ID names this embedder for the DB marker. The dimension is part of it
// because a HashEmbedder of another width is a different vector space.
func (h *HashEmbedder) ID() string { return fmt.Sprintf("hash@%d", h.dim) }

func (h *HashEmbedder) Close() error { return nil }

func (h *HashEmbedder) Embed(_ context.Context, text string) ([]float32, error) {
	v := make([]float32, h.dim)
	for _, tok := range tokenize(text) {
		idx := fnv1a(tok) % uint32(h.dim)
		v[idx] += 1.0
	}
	var sum float64
	for _, x := range v {
		sum += float64(x) * float64(x)
	}
	if sum == 0 {
		return v, nil
	}
	inv := float32(1.0 / math.Sqrt(sum))
	for i := range v {
		v[i] *= inv
	}
	return v, nil
}

func fnv1a(s string) uint32 {
	h := uint32(2166136261)
	for i := 0; i < len(s); i++ {
		h ^= uint32(s[i])
		h *= 16777619
	}
	return h
}

// tokenize — lowercase, split on non-letter/digit, drop empties + 1-char noise.
// Rune-based and Unicode-aware: Maven is ru-first, so a byte-only ASCII filter
// would drop every Cyrillic word (its bytes are all ≥ 0x80) and embed Russian
// utterances to the zero vector — cosine 0 across all intents, misrouting every
// RU command. unicode.IsLetter covers Cyrillic + Latin; the real ONNX model
// brings its own tokenizer.
func tokenize(s string) []string {
	out := make([]string, 0, 8)
	var b []rune
	flush := func() {
		if len(b) > 1 {
			out = append(out, string(b))
		}
		b = b[:0]
	}
	for _, r := range s {
		if unicode.IsLetter(r) || unicode.IsDigit(r) {
			b = append(b, unicode.ToLower(r))
		} else {
			flush()
		}
	}
	flush()
	return out
}