initial commit

internal/router/embedder_test.go

@@ -0,0 +1,39 @@
+package router
+
+import (
+	"context"
+	"testing"
+)
+
+// TestHashEmbedderCyrillic guards the ru-first floor: a byte-only tokenizer
+// drops every Cyrillic word (bytes ≥ 0x80) and embeds Russian to the zero
+// vector — cosine 0 across all intents, misrouting every RU utterance. Assert
+// non-zero vectors, and that shared Russian words produce more similar vectors
+// than disjoint ones (the point of the bag-of-words floor).
+func TestHashEmbedderCyrillic(t *testing.T) {
+	e := NewHashEmbedder(1024)
+	ctx := context.Background()
+
+	nonZero := func(text string) []float32 {
+		v, err := e.Embed(ctx, text)
+		if err != nil {
+			t.Fatalf("embed %q: %v", text, err)
+		}
+		var sum float64
+		for _, x := range v {
+			sum += float64(x) * float64(x)
+		}
+		if sum == 0 {
+			t.Fatalf("embed %q → zero vector (tokenizer dropped all tokens)", text)
+		}
+		return v
+	}
+
+	a := nonZero("найди заметку про сервер")
+	b := nonZero("найди заметку про роутер") // shares 3 of 4 words
+	c := nonZero("перезагрузи компьютер")    // disjoint
+
+	if cosine(a, b) <= cosine(a, c) {
+		t.Fatalf("cosine(shared)=%.3f not > cosine(disjoint)=%.3f", cosine(a, b), cosine(a, c))
+	}
+}