package router import ( "context" "fmt" "math" "unicode" ) // Embedder produces a dense vector for a text utterance. The classifier is // nearest-centroid over labeled-intent example vectors; the embedder is the // one impure seam — the production path is the multilingual ONNX int8 model // (multilingual-e5-small or paraphrase-multilingual-MiniLM-L12-v2, ~120mb, // bilingual ru+en native, no separate ru/en path). That model is a later // module; anything deterministic + dimension-fixed works here, which keeps // the classifier + cascade unit-testable without the model loaded. type Embedder interface { Dim() int Embed(ctx context.Context, text string) ([]float32, error) Close() error } // IdentifiedEmbedder — an embedder that can name itself. The name goes into // the DB next to the vectors it wrote, so a later model swap is caught instead // of silently returning nonsense scores (Vikunja #378). type IdentifiedEmbedder interface { Embedder ID() string } // EmbedderID is the stable string stored alongside the vectors. It comes from // the embedder itself — nobody hand-types a model name twice — and changes // whenever the model or its dimension changes. func EmbedderID(e Embedder) string { if i, ok := e.(IdentifiedEmbedder); ok { return i.ID() } return fmt.Sprintf("unknown@%d", e.Dim()) } // AsymmetricEmbedder — an embedder that wants to know whether a text is a // search query or a stored passage. Recall is asymmetric: a short question // goes in, a longer note comes out. The e5 family is trained for exactly that // and needs the side written into the text ("query: " / "passage: "). // // Optional on purpose: HashEmbedder has no such notion, so callers go through // EmbedQuery and EmbedPassage below, which fall back to plain Embed. type AsymmetricEmbedder interface { Embedder EmbedQuery(ctx context.Context, text string) ([]float32, error) EmbedPassage(ctx context.Context, text string) ([]float32, error) } // EmbedQuery embeds text that is being searched WITH — a question. func EmbedQuery(ctx context.Context, e Embedder, text string) ([]float32, error) { if a, ok := e.(AsymmetricEmbedder); ok { return a.EmbedQuery(ctx, text) } return e.Embed(ctx, text) } // EmbedPassage embeds text that is being searched FOR — a note or a fact on // its way into the store. Store and lookup must use these two calls, not one // of them twice, or the asymmetry buys nothing. func EmbedPassage(ctx context.Context, e Embedder, text string) ([]float32, error) { if a, ok := e.(AsymmetricEmbedder); ok { return a.EmbedPassage(ctx, text) } return e.Embed(ctx, text) } // HashEmbedder — a deterministic bag-of-words embedder used for tests and as a // non-zero default floor. NOT semantically meaningful across languages; the // real classifier swaps in the multilingual ONNX model wholesale. // // Token collisions are the point: same surface words ⇒ similar vectors ⇒ the // centroid math is testable. Each token hashes into a dimension; weights // accumulate then L2-normalize so cosine similarity is a clean inner product. type HashEmbedder struct { dim int } func NewHashEmbedder(dim int) *HashEmbedder { if dim <= 0 { dim = 128 } return &HashEmbedder{dim: dim} } func (h *HashEmbedder) Dim() int { return h.dim } // ID names this embedder for the DB marker. The dimension is part of it // because a HashEmbedder of another width is a different vector space. func (h *HashEmbedder) ID() string { return fmt.Sprintf("hash@%d", h.dim) } func (h *HashEmbedder) Close() error { return nil } func (h *HashEmbedder) Embed(_ context.Context, text string) ([]float32, error) { v := make([]float32, h.dim) for _, tok := range tokenize(text) { idx := fnv1a(tok) % uint32(h.dim) v[idx] += 1.0 } var sum float64 for _, x := range v { sum += float64(x) * float64(x) } if sum == 0 { return v, nil } inv := float32(1.0 / math.Sqrt(sum)) for i := range v { v[i] *= inv } return v, nil } func fnv1a(s string) uint32 { h := uint32(2166136261) for i := 0; i < len(s); i++ { h ^= uint32(s[i]) h *= 16777619 } return h } // tokenize — lowercase, split on non-letter/digit, drop empties + 1-char noise. // Rune-based and Unicode-aware: Maven is ru-first, so a byte-only ASCII filter // would drop every Cyrillic word (its bytes are all ≥ 0x80) and embed Russian // utterances to the zero vector — cosine 0 across all intents, misrouting every // RU command. unicode.IsLetter covers Cyrillic + Latin; the real ONNX model // brings its own tokenizer. func tokenize(s string) []string { out := make([]string, 0, 8) var b []rune flush := func() { if len(b) > 1 { out = append(out, string(b)) } b = b[:0] } for _, r := range s { if unicode.IsLetter(r) || unicode.IsDigit(r) { b = append(b, unicode.ToLower(r)) } else { flush() } } flush() return out }