claude
e9049ad27e
17 tests covering: whitespace collapse, case folding, NFKC normalization, punctuation preservation, mixed-script identifiers, ё passthrough, and property invariants (idempotent, deterministic, never removes punctuation/ wake words/numbers, never mutates original). Test-only foldYo helper proves ё→е is lossy (всё → все) without exposing an unused production function. Vikunja: #725
316 lines
8.2 KiB
Go
316 lines
8.2 KiB
Go
package router
|
||
|
||
import (
|
||
"strings"
|
||
"testing"
|
||
"unicode"
|
||
)
|
||
|
||
// --- NormalizeMatchText tests ---
|
||
|
||
func TestNormalizeMatchTextWhitespace(t *testing.T) {
|
||
tests := []struct {
|
||
in string
|
||
want string
|
||
}{
|
||
{" привет ", "привет"},
|
||
{"привет\t\tмир", "привет мир"},
|
||
{"привет\nмир", "привет мир"},
|
||
{"привет\r\nмир", "привет мир"},
|
||
{"привет \t мир", "привет мир"},
|
||
{" ", ""},
|
||
{"", ""},
|
||
{"привет", "привет"},
|
||
}
|
||
for _, tt := range tests {
|
||
got := NormalizeMatchText(tt.in)
|
||
if got != tt.want {
|
||
t.Errorf("NormalizeMatchText(%q) = %q, want %q", tt.in, got, tt.want)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestNormalizeMatchTextCase(t *testing.T) {
|
||
tests := []struct {
|
||
in string
|
||
want string
|
||
}{
|
||
{"ПрИвЕт MAVEN", "привет maven"},
|
||
{"ПРИВЕТ", "привет"},
|
||
{"hello", "hello"},
|
||
{"HELLO", "hello"},
|
||
{"Мэйвен", "мэйвен"},
|
||
{"MAVEN", "maven"},
|
||
}
|
||
for _, tt := range tests {
|
||
got := NormalizeMatchText(tt.in)
|
||
if got != tt.want {
|
||
t.Errorf("NormalizeMatchText(%q) = %q, want %q", tt.in, got, tt.want)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestNormalizeMatchTextNFKC(t *testing.T) {
|
||
tests := []struct {
|
||
in string
|
||
want string
|
||
}{
|
||
// Fullwidth Latin letters → ASCII
|
||
{"\uff28\uff45\uff4c\uff4c\uff4f", "hello"},
|
||
// Superscript digits → ASCII
|
||
{"2\u00b9\u00b2\u00b3", "2123"},
|
||
// Compatible ligature
|
||
{"\ufb01", "fi"},
|
||
// Cyrillic compact forms
|
||
{"\u0439", "\u0439"}, // already NFC, stays as-is
|
||
}
|
||
for _, tt := range tests {
|
||
got := NormalizeMatchText(tt.in)
|
||
if got != tt.want {
|
||
t.Errorf("NormalizeMatchText(%q) = %q, want %q", tt.in, got, tt.want)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestNormalizeMatchTextPunctuationPreserved(t *testing.T) {
|
||
tests := []struct {
|
||
in string
|
||
want string
|
||
}{
|
||
{"это вопрос?", "это вопрос?"},
|
||
{"привет!", "привет!"},
|
||
{"да.", "да."},
|
||
{"напомни: через час", "напомни: через час"},
|
||
{"maven, restart nginx", "maven, restart nginx"},
|
||
{"\"quoted text\"", "\"quoted text\""},
|
||
}
|
||
for _, tt := range tests {
|
||
got := NormalizeMatchText(tt.in)
|
||
if got != tt.want {
|
||
t.Errorf("NormalizeMatchText(%q) = %q, want %q", tt.in, got, tt.want)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestNormalizeMatchTextMixedScript(t *testing.T) {
|
||
tests := []struct {
|
||
in string
|
||
want string
|
||
}{
|
||
{"restart сервер", "restart сервер"},
|
||
{"nginx на сервере", "nginx на сервере"},
|
||
{"MAC-адрес", "mac-адрес"},
|
||
{"token123", "token123"},
|
||
{"path/to/file", "path/to/file"},
|
||
}
|
||
for _, tt := range tests {
|
||
got := NormalizeMatchText(tt.in)
|
||
if got != tt.want {
|
||
t.Errorf("NormalizeMatchText(%q) = %q, want %q", tt.in, got, tt.want)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestNormalizeMatchTextYoPreserved(t *testing.T) {
|
||
// ё is NOT folded by NormalizeMatchText — it passes through as a letter.
|
||
tests := []struct {
|
||
in string
|
||
want string
|
||
}{
|
||
{"всё", "всё"},
|
||
{"ВсЁ", "всё"},
|
||
{"ещё", "ещё"},
|
||
{"Ещё", "ещё"},
|
||
}
|
||
for _, tt := range tests {
|
||
got := NormalizeMatchText(tt.in)
|
||
if got != tt.want {
|
||
t.Errorf("NormalizeMatchText(%q) = %q, want %q", tt.in, got, tt.want)
|
||
}
|
||
}
|
||
}
|
||
|
||
// --- FoldYo tests ---
|
||
|
||
// foldYo is a test-only helper mapping ё→е and Ё→Е. The fold is lossy:
|
||
// "всё" and "все" are distinct Russian words, and folding destroys that
|
||
// distinction. Not exposed in production code.
|
||
func foldYo(in string) string {
|
||
var b strings.Builder
|
||
b.Grow(len(in))
|
||
for _, r := range in {
|
||
switch r {
|
||
case 'ё':
|
||
b.WriteRune('е')
|
||
case 'Ё':
|
||
b.WriteRune('Е')
|
||
default:
|
||
b.WriteRune(r)
|
||
}
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
func TestFoldYoBasic(t *testing.T) {
|
||
tests := []struct {
|
||
in string
|
||
want string
|
||
}{
|
||
{"всё", "все"},
|
||
{"ВсЁ", "ВсЕ"}, // ё→е, Ё→Е (uppercase preserves case)
|
||
{"ещё", "еще"},
|
||
{"Ещё", "Еще"},
|
||
{"ёж", "еж"},
|
||
{"Ёж", "Еж"},
|
||
{"привет", "привет"}, // no ё
|
||
{"", ""},
|
||
}
|
||
for _, tt := range tests {
|
||
got := foldYo(tt.in)
|
||
if got != tt.want {
|
||
t.Errorf("foldYo(%q) = %q, want %q", tt.in, got, tt.want)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestFoldYoIsLossy(t *testing.T) {
|
||
// "всё" (everything) and "все" (everyone/all) are distinct words.
|
||
// Folding makes them identical — this test pins that the loss is real.
|
||
a := foldYo("всё")
|
||
b := "все"
|
||
if a != b {
|
||
t.Errorf("foldYo(\"всё\") = %q, want %q (fold should produce identical output)", a, b)
|
||
}
|
||
// The original pair must be distinct.
|
||
if "всё" == "все" {
|
||
t.Error("всё and все should be distinct strings before folding")
|
||
}
|
||
}
|
||
|
||
func TestFoldYoIdempotent(t *testing.T) {
|
||
inputs := []string{"всё", "Ещё", "Ёж", "привет", ""}
|
||
for _, in := range inputs {
|
||
first := foldYo(in)
|
||
second := foldYo(first)
|
||
if first != second {
|
||
t.Errorf("foldYo not idempotent: foldYo(%q) = %q, foldYo(that) = %q", in, first, second)
|
||
}
|
||
}
|
||
}
|
||
|
||
// --- Property / invariant tests ---
|
||
|
||
func TestNormalizeMatchTextIdempotent(t *testing.T) {
|
||
inputs := []string{
|
||
" привет ",
|
||
"ПрИвЕт MAVEN",
|
||
"это вопрос?",
|
||
"всё",
|
||
"restart nginx",
|
||
"",
|
||
"\t\n",
|
||
"\uff28\uff45\uff4c\uff4c\uff4f", // NFKC fullwidth
|
||
}
|
||
for _, in := range inputs {
|
||
first := NormalizeMatchText(in)
|
||
second := NormalizeMatchText(first)
|
||
if first != second {
|
||
t.Errorf("NormalizeMatchText not idempotent on %q: first=%q, second=%q", in, first, second)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestNormalizeMatchTextNeverRemovesPunctuation(t *testing.T) {
|
||
inputs := []string{
|
||
"это вопрос?",
|
||
"привет!",
|
||
"да.",
|
||
"напомни: через час",
|
||
"maven, restart nginx",
|
||
}
|
||
for _, in := range inputs {
|
||
got := NormalizeMatchText(in)
|
||
// Every ASCII punctuation char in the input must appear in the output.
|
||
for _, r := range in {
|
||
if r > 0x20 && r < 0x7f && !unicode.IsLetter(r) && !unicode.IsDigit(r) && !unicode.IsSpace(r) {
|
||
if !strings.ContainsRune(got, r) {
|
||
t.Errorf("NormalizeMatchText(%q) lost punctuation %c → %q", in, r, got)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestNormalizeMatchTextNeverRemovesWakeWord(t *testing.T) {
|
||
inputs := []string{
|
||
"maven restart nginx",
|
||
"Мэйвен который час",
|
||
"мавен напомни",
|
||
}
|
||
for _, in := range inputs {
|
||
got := NormalizeMatchText(in)
|
||
if strings.TrimSpace(got) == "" {
|
||
t.Errorf("NormalizeMatchText(%q) produced empty output", in)
|
||
}
|
||
// The wake word should still be present as a substring (lowercased).
|
||
lowered := strings.ToLower(in)
|
||
for _, wake := range []string{"maven", "мэйвен", "мейвен", "майвен", "мавен", "мавена", "мавену", "мавеном"} {
|
||
if strings.Contains(lowered, wake) && !strings.Contains(got, wake) {
|
||
t.Errorf("NormalizeMatchText(%q) lost wake word %q → %q", in, wake, got)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestNormalizeMatchTextNeverRewritesNumbers(t *testing.T) {
|
||
inputs := []string{
|
||
"семь вечера",
|
||
"три часа",
|
||
"через два часа",
|
||
}
|
||
for _, in := range inputs {
|
||
got := NormalizeMatchText(in)
|
||
// Number words must not become digits.
|
||
if strings.ContainsAny(got, "0123456789") {
|
||
t.Errorf("NormalizeMatchText(%q) introduced digits → %q", in, got)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestNormalizeMatchTextNeverMutatesOriginal(t *testing.T) {
|
||
inputs := []string{" привет ", "ПрИвЕт", "это вопрос?"}
|
||
for _, in := range inputs {
|
||
orig := in
|
||
_ = NormalizeMatchText(in)
|
||
if in != orig {
|
||
t.Errorf("NormalizeMatchText mutated input: was %q, now %q", orig, in)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestNormalizeMatchTextIsDeterministic(t *testing.T) {
|
||
input := " ПрИвЕт MAVEN "
|
||
a := NormalizeMatchText(input)
|
||
b := NormalizeMatchText(input)
|
||
if a != b {
|
||
t.Errorf("NormalizeMatchText not deterministic: %q vs %q", a, b)
|
||
}
|
||
}
|
||
|
||
func TestMatchTextDarkDataInvariant(t *testing.T) {
|
||
// MatchText is currently not authoritative for any routing/action result.
|
||
// Pin this by verifying that constructing a NormalizedInput with MatchText
|
||
// does not change the Text field.
|
||
input := NormalizedInput{
|
||
Text: "Привет",
|
||
MatchText: NormalizeMatchText("Привет"),
|
||
Source: InputSourceText,
|
||
}
|
||
if input.Text != "Привет" {
|
||
t.Errorf("Text was mutated to %q", input.Text)
|
||
}
|
||
if input.MatchText != "привет" {
|
||
t.Errorf("MatchText = %q, want %q", input.MatchText, "привет")
|
||
}
|
||
}
|