package router import ( "context" "testing" ) // TestHashEmbedderCyrillic guards the ru-first floor: a byte-only tokenizer // drops every Cyrillic word (bytes ≥ 0x80) and embeds Russian to the zero // vector — cosine 0 across all intents, misrouting every RU utterance. Assert // non-zero vectors, and that shared Russian words produce more similar vectors // than disjoint ones (the point of the bag-of-words floor). func TestHashEmbedderCyrillic(t *testing.T) { e := NewHashEmbedder(1024) ctx := context.Background() nonZero := func(text string) []float32 { v, err := e.Embed(ctx, text) if err != nil { t.Fatalf("embed %q: %v", text, err) } var sum float64 for _, x := range v { sum += float64(x) * float64(x) } if sum == 0 { t.Fatalf("embed %q → zero vector (tokenizer dropped all tokens)", text) } return v } a := nonZero("найди заметку про сервер") b := nonZero("найди заметку про роутер") // shares 3 of 4 words c := nonZero("перезагрузи компьютер") // disjoint if cosine(a, b) <= cosine(a, c) { t.Fatalf("cosine(shared)=%.3f not > cosine(disjoint)=%.3f", cosine(a, b), cosine(a, c)) } }