kami
40 lines
1.2 KiB
Go
40 lines
1.2 KiB
Go
package router
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
)
|
|
|
|
// TestHashEmbedderCyrillic guards the ru-first floor: a byte-only tokenizer
|
|
// drops every Cyrillic word (bytes ≥ 0x80) and embeds Russian to the zero
|
|
// vector — cosine 0 across all intents, misrouting every RU utterance. Assert
|
|
// non-zero vectors, and that shared Russian words produce more similar vectors
|
|
// than disjoint ones (the point of the bag-of-words floor).
|
|
func TestHashEmbedderCyrillic(t *testing.T) {
|
|
e := NewHashEmbedder(1024)
|
|
ctx := context.Background()
|
|
|
|
nonZero := func(text string) []float32 {
|
|
v, err := e.Embed(ctx, text)
|
|
if err != nil {
|
|
t.Fatalf("embed %q: %v", text, err)
|
|
}
|
|
var sum float64
|
|
for _, x := range v {
|
|
sum += float64(x) * float64(x)
|
|
}
|
|
if sum == 0 {
|
|
t.Fatalf("embed %q → zero vector (tokenizer dropped all tokens)", text)
|
|
}
|
|
return v
|
|
}
|
|
|
|
a := nonZero("найди заметку про сервер")
|
|
b := nonZero("найди заметку про роутер") // shares 3 of 4 words
|
|
c := nonZero("перезагрузи компьютер") // disjoint
|
|
|
|
if cosine(a, b) <= cosine(a, c) {
|
|
t.Fatalf("cosine(shared)=%.3f not > cosine(disjoint)=%.3f", cosine(a, b), cosine(a, c))
|
|
}
|
|
}
|