kami
98 lines
2.7 KiB
Go
98 lines
2.7 KiB
Go
package router
|
|
|
|
import (
|
|
"context"
|
|
"math"
|
|
"unicode"
|
|
)
|
|
|
|
// Embedder produces a dense vector for a text utterance. The classifier is
|
|
// nearest-centroid over labeled-intent example vectors; the embedder is the
|
|
// one impure seam — the production path is the multilingual ONNX int8 model
|
|
// (multilingual-e5-small or paraphrase-multilingual-MiniLM-L12-v2, ~120mb,
|
|
// bilingual ru+en native, no separate ru/en path). That model is a later
|
|
// module; anything deterministic + dimension-fixed works here, which keeps
|
|
// the classifier + cascade unit-testable without the model loaded.
|
|
type Embedder interface {
|
|
Dim() int
|
|
Embed(ctx context.Context, text string) ([]float32, error)
|
|
Close() error
|
|
}
|
|
|
|
// HashEmbedder — a deterministic bag-of-words embedder used for tests and as a
|
|
// non-zero default floor. NOT semantically meaningful across languages; the
|
|
// real classifier swaps in the multilingual ONNX model wholesale.
|
|
//
|
|
// Token collisions are the point: same surface words ⇒ similar vectors ⇒ the
|
|
// centroid math is testable. Each token hashes into a dimension; weights
|
|
// accumulate then L2-normalize so cosine similarity is a clean inner product.
|
|
type HashEmbedder struct {
|
|
dim int
|
|
}
|
|
|
|
func NewHashEmbedder(dim int) *HashEmbedder {
|
|
if dim <= 0 {
|
|
dim = 128
|
|
}
|
|
return &HashEmbedder{dim: dim}
|
|
}
|
|
|
|
func (h *HashEmbedder) Dim() int { return h.dim }
|
|
|
|
func (h *HashEmbedder) Close() error { return nil }
|
|
|
|
func (h *HashEmbedder) Embed(_ context.Context, text string) ([]float32, error) {
|
|
v := make([]float32, h.dim)
|
|
for _, tok := range tokenize(text) {
|
|
idx := fnv1a(tok) % uint32(h.dim)
|
|
v[idx] += 1.0
|
|
}
|
|
var sum float64
|
|
for _, x := range v {
|
|
sum += float64(x) * float64(x)
|
|
}
|
|
if sum == 0 {
|
|
return v, nil
|
|
}
|
|
inv := float32(1.0 / math.Sqrt(sum))
|
|
for i := range v {
|
|
v[i] *= inv
|
|
}
|
|
return v, nil
|
|
}
|
|
|
|
func fnv1a(s string) uint32 {
|
|
h := uint32(2166136261)
|
|
for i := 0; i < len(s); i++ {
|
|
h ^= uint32(s[i])
|
|
h *= 16777619
|
|
}
|
|
return h
|
|
}
|
|
|
|
// tokenize — lowercase, split on non-letter/digit, drop empties + 1-char noise.
|
|
// Rune-based and Unicode-aware: Maven is ru-first, so a byte-only ASCII filter
|
|
// would drop every Cyrillic word (its bytes are all ≥ 0x80) and embed Russian
|
|
// utterances to the zero vector — cosine 0 across all intents, misrouting every
|
|
// RU command. unicode.IsLetter covers Cyrillic + Latin; the real ONNX model
|
|
// brings its own tokenizer.
|
|
func tokenize(s string) []string {
|
|
out := make([]string, 0, 8)
|
|
var b []rune
|
|
flush := func() {
|
|
if len(b) > 1 {
|
|
out = append(out, string(b))
|
|
}
|
|
b = b[:0]
|
|
}
|
|
for _, r := range s {
|
|
if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
|
b = append(b, unicode.ToLower(r))
|
|
} else {
|
|
flush()
|
|
}
|
|
}
|
|
flush()
|
|
return out
|
|
}
|