initial commit

internal/router/embedder.go

@@ -0,0 +1,97 @@
+package router
+
+import (
+	"context"
+	"math"
+	"unicode"
+)
+
+// Embedder produces a dense vector for a text utterance. The classifier is
+// nearest-centroid over labeled-intent example vectors; the embedder is the
+// one impure seam — the production path is the multilingual ONNX int8 model
+// (multilingual-e5-small or paraphrase-multilingual-MiniLM-L12-v2, ~120mb,
+// bilingual ru+en native, no separate ru/en path). That model is a later
+// module; anything deterministic + dimension-fixed works here, which keeps
+// the classifier + cascade unit-testable without the model loaded.
+type Embedder interface {
+	Dim() int
+	Embed(ctx context.Context, text string) ([]float32, error)
+	Close() error
+}
+
+// HashEmbedder — a deterministic bag-of-words embedder used for tests and as a
+// non-zero default floor. NOT semantically meaningful across languages; the
+// real classifier swaps in the multilingual ONNX model wholesale.
+//
+// Token collisions are the point: same surface words ⇒ similar vectors ⇒ the
+// centroid math is testable. Each token hashes into a dimension; weights
+// accumulate then L2-normalize so cosine similarity is a clean inner product.
+type HashEmbedder struct {
+	dim int
+}
+
+func NewHashEmbedder(dim int) *HashEmbedder {
+	if dim <= 0 {
+		dim = 128
+	}
+	return &HashEmbedder{dim: dim}
+}
+
+func (h *HashEmbedder) Dim() int { return h.dim }
+
+func (h *HashEmbedder) Close() error { return nil }
+
+func (h *HashEmbedder) Embed(_ context.Context, text string) ([]float32, error) {
+	v := make([]float32, h.dim)
+	for _, tok := range tokenize(text) {
+		idx := fnv1a(tok) % uint32(h.dim)
+		v[idx] += 1.0
+	}
+	var sum float64
+	for _, x := range v {
+		sum += float64(x) * float64(x)
+	}
+	if sum == 0 {
+		return v, nil
+	}
+	inv := float32(1.0 / math.Sqrt(sum))
+	for i := range v {
+		v[i] *= inv
+	}
+	return v, nil
+}
+
+func fnv1a(s string) uint32 {
+	h := uint32(2166136261)
+	for i := 0; i < len(s); i++ {
+		h ^= uint32(s[i])
+		h *= 16777619
+	}
+	return h
+}
+
+// tokenize — lowercase, split on non-letter/digit, drop empties + 1-char noise.
+// Rune-based and Unicode-aware: Maven is ru-first, so a byte-only ASCII filter
+// would drop every Cyrillic word (its bytes are all ≥ 0x80) and embed Russian
+// utterances to the zero vector — cosine 0 across all intents, misrouting every
+// RU command. unicode.IsLetter covers Cyrillic + Latin; the real ONNX model
+// brings its own tokenizer.
+func tokenize(s string) []string {
+	out := make([]string, 0, 8)
+	var b []rune
+	flush := func() {
+		if len(b) > 1 {
+			out = append(out, string(b))
+		}
+		b = b[:0]
+	}
+	for _, r := range s {
+		if unicode.IsLetter(r) || unicode.IsDigit(r) {
+			b = append(b, unicode.ToLower(r))
+		} else {
+			flush()
+		}
+	}
+	flush()
+	return out
+}