router: add NormalizeMatchText tests and property invariants (slice 9b)
17 tests covering: whitespace collapse, case folding, NFKC normalization, punctuation preservation, mixed-script identifiers, ё passthrough, and property invariants (idempotent, deterministic, never removes punctuation/ wake words/numbers, never mutates original). Test-only foldYo helper proves ё→е is lossy (всё → все) without exposing an unused production function. Vikunja: #725
This commit is contained in:
@@ -7,19 +7,22 @@ import (
|
||||
)
|
||||
|
||||
// TestNormalizedInputIsMinimalValueObject — the typed ingress boundary carries
|
||||
// text and source and nothing else. This test pins the shape so a future slice
|
||||
// text, match text and source. This test pins the shape so a future slice
|
||||
// cannot add fields without updating every construction site.
|
||||
func TestNormalizedInputIsMinimalValueObject(t *testing.T) {
|
||||
input := NormalizedInput{Text: "привет", Source: InputSourceVoice}
|
||||
input := NormalizedInput{Text: "привет", MatchText: "привет", Source: InputSourceVoice}
|
||||
if input.Text != "привет" {
|
||||
t.Errorf("Text = %q, want %q", input.Text, "привет")
|
||||
}
|
||||
if input.MatchText != "привет" {
|
||||
t.Errorf("MatchText = %q, want %q", input.MatchText, "привет")
|
||||
}
|
||||
if input.Source != InputSourceVoice {
|
||||
t.Errorf("Source = %q, want %q", input.Source, InputSourceVoice)
|
||||
}
|
||||
// Empty zero value is usable.
|
||||
var zero NormalizedInput
|
||||
if zero.Text != "" || zero.Source != "" {
|
||||
if zero.Text != "" || zero.MatchText != "" || zero.Source != "" {
|
||||
t.Errorf("zero value is not empty: %+v", zero)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,315 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// --- NormalizeMatchText tests ---
|
||||
|
||||
func TestNormalizeMatchTextWhitespace(t *testing.T) {
|
||||
tests := []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{" привет ", "привет"},
|
||||
{"привет\t\tмир", "привет мир"},
|
||||
{"привет\nмир", "привет мир"},
|
||||
{"привет\r\nмир", "привет мир"},
|
||||
{"привет \t мир", "привет мир"},
|
||||
{" ", ""},
|
||||
{"", ""},
|
||||
{"привет", "привет"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := NormalizeMatchText(tt.in)
|
||||
if got != tt.want {
|
||||
t.Errorf("NormalizeMatchText(%q) = %q, want %q", tt.in, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeMatchTextCase(t *testing.T) {
|
||||
tests := []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"ПрИвЕт MAVEN", "привет maven"},
|
||||
{"ПРИВЕТ", "привет"},
|
||||
{"hello", "hello"},
|
||||
{"HELLO", "hello"},
|
||||
{"Мэйвен", "мэйвен"},
|
||||
{"MAVEN", "maven"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := NormalizeMatchText(tt.in)
|
||||
if got != tt.want {
|
||||
t.Errorf("NormalizeMatchText(%q) = %q, want %q", tt.in, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeMatchTextNFKC(t *testing.T) {
|
||||
tests := []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
// Fullwidth Latin letters → ASCII
|
||||
{"\uff28\uff45\uff4c\uff4c\uff4f", "hello"},
|
||||
// Superscript digits → ASCII
|
||||
{"2\u00b9\u00b2\u00b3", "2123"},
|
||||
// Compatible ligature
|
||||
{"\ufb01", "fi"},
|
||||
// Cyrillic compact forms
|
||||
{"\u0439", "\u0439"}, // already NFC, stays as-is
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := NormalizeMatchText(tt.in)
|
||||
if got != tt.want {
|
||||
t.Errorf("NormalizeMatchText(%q) = %q, want %q", tt.in, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeMatchTextPunctuationPreserved(t *testing.T) {
|
||||
tests := []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"это вопрос?", "это вопрос?"},
|
||||
{"привет!", "привет!"},
|
||||
{"да.", "да."},
|
||||
{"напомни: через час", "напомни: через час"},
|
||||
{"maven, restart nginx", "maven, restart nginx"},
|
||||
{"\"quoted text\"", "\"quoted text\""},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := NormalizeMatchText(tt.in)
|
||||
if got != tt.want {
|
||||
t.Errorf("NormalizeMatchText(%q) = %q, want %q", tt.in, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeMatchTextMixedScript(t *testing.T) {
|
||||
tests := []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"restart сервер", "restart сервер"},
|
||||
{"nginx на сервере", "nginx на сервере"},
|
||||
{"MAC-адрес", "mac-адрес"},
|
||||
{"token123", "token123"},
|
||||
{"path/to/file", "path/to/file"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := NormalizeMatchText(tt.in)
|
||||
if got != tt.want {
|
||||
t.Errorf("NormalizeMatchText(%q) = %q, want %q", tt.in, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeMatchTextYoPreserved(t *testing.T) {
|
||||
// ё is NOT folded by NormalizeMatchText — it passes through as a letter.
|
||||
tests := []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"всё", "всё"},
|
||||
{"ВсЁ", "всё"},
|
||||
{"ещё", "ещё"},
|
||||
{"Ещё", "ещё"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := NormalizeMatchText(tt.in)
|
||||
if got != tt.want {
|
||||
t.Errorf("NormalizeMatchText(%q) = %q, want %q", tt.in, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- FoldYo tests ---
|
||||
|
||||
// foldYo is a test-only helper mapping ё→е and Ё→Е. The fold is lossy:
|
||||
// "всё" and "все" are distinct Russian words, and folding destroys that
|
||||
// distinction. Not exposed in production code.
|
||||
func foldYo(in string) string {
|
||||
var b strings.Builder
|
||||
b.Grow(len(in))
|
||||
for _, r := range in {
|
||||
switch r {
|
||||
case 'ё':
|
||||
b.WriteRune('е')
|
||||
case 'Ё':
|
||||
b.WriteRune('Е')
|
||||
default:
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func TestFoldYoBasic(t *testing.T) {
|
||||
tests := []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"всё", "все"},
|
||||
{"ВсЁ", "ВсЕ"}, // ё→е, Ё→Е (uppercase preserves case)
|
||||
{"ещё", "еще"},
|
||||
{"Ещё", "Еще"},
|
||||
{"ёж", "еж"},
|
||||
{"Ёж", "Еж"},
|
||||
{"привет", "привет"}, // no ё
|
||||
{"", ""},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := foldYo(tt.in)
|
||||
if got != tt.want {
|
||||
t.Errorf("foldYo(%q) = %q, want %q", tt.in, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFoldYoIsLossy(t *testing.T) {
|
||||
// "всё" (everything) and "все" (everyone/all) are distinct words.
|
||||
// Folding makes them identical — this test pins that the loss is real.
|
||||
a := foldYo("всё")
|
||||
b := "все"
|
||||
if a != b {
|
||||
t.Errorf("foldYo(\"всё\") = %q, want %q (fold should produce identical output)", a, b)
|
||||
}
|
||||
// The original pair must be distinct.
|
||||
if "всё" == "все" {
|
||||
t.Error("всё and все should be distinct strings before folding")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFoldYoIdempotent(t *testing.T) {
|
||||
inputs := []string{"всё", "Ещё", "Ёж", "привет", ""}
|
||||
for _, in := range inputs {
|
||||
first := foldYo(in)
|
||||
second := foldYo(first)
|
||||
if first != second {
|
||||
t.Errorf("foldYo not idempotent: foldYo(%q) = %q, foldYo(that) = %q", in, first, second)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Property / invariant tests ---
|
||||
|
||||
func TestNormalizeMatchTextIdempotent(t *testing.T) {
|
||||
inputs := []string{
|
||||
" привет ",
|
||||
"ПрИвЕт MAVEN",
|
||||
"это вопрос?",
|
||||
"всё",
|
||||
"restart nginx",
|
||||
"",
|
||||
"\t\n",
|
||||
"\uff28\uff45\uff4c\uff4c\uff4f", // NFKC fullwidth
|
||||
}
|
||||
for _, in := range inputs {
|
||||
first := NormalizeMatchText(in)
|
||||
second := NormalizeMatchText(first)
|
||||
if first != second {
|
||||
t.Errorf("NormalizeMatchText not idempotent on %q: first=%q, second=%q", in, first, second)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeMatchTextNeverRemovesPunctuation(t *testing.T) {
|
||||
inputs := []string{
|
||||
"это вопрос?",
|
||||
"привет!",
|
||||
"да.",
|
||||
"напомни: через час",
|
||||
"maven, restart nginx",
|
||||
}
|
||||
for _, in := range inputs {
|
||||
got := NormalizeMatchText(in)
|
||||
// Every ASCII punctuation char in the input must appear in the output.
|
||||
for _, r := range in {
|
||||
if r > 0x20 && r < 0x7f && !unicode.IsLetter(r) && !unicode.IsDigit(r) && !unicode.IsSpace(r) {
|
||||
if !strings.ContainsRune(got, r) {
|
||||
t.Errorf("NormalizeMatchText(%q) lost punctuation %c → %q", in, r, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeMatchTextNeverRemovesWakeWord(t *testing.T) {
|
||||
inputs := []string{
|
||||
"maven restart nginx",
|
||||
"Мэйвен который час",
|
||||
"мавен напомни",
|
||||
}
|
||||
for _, in := range inputs {
|
||||
got := NormalizeMatchText(in)
|
||||
if strings.TrimSpace(got) == "" {
|
||||
t.Errorf("NormalizeMatchText(%q) produced empty output", in)
|
||||
}
|
||||
// The wake word should still be present as a substring (lowercased).
|
||||
lowered := strings.ToLower(in)
|
||||
for _, wake := range []string{"maven", "мэйвен", "мейвен", "майвен", "мавен", "мавена", "мавену", "мавеном"} {
|
||||
if strings.Contains(lowered, wake) && !strings.Contains(got, wake) {
|
||||
t.Errorf("NormalizeMatchText(%q) lost wake word %q → %q", in, wake, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeMatchTextNeverRewritesNumbers(t *testing.T) {
|
||||
inputs := []string{
|
||||
"семь вечера",
|
||||
"три часа",
|
||||
"через два часа",
|
||||
}
|
||||
for _, in := range inputs {
|
||||
got := NormalizeMatchText(in)
|
||||
// Number words must not become digits.
|
||||
if strings.ContainsAny(got, "0123456789") {
|
||||
t.Errorf("NormalizeMatchText(%q) introduced digits → %q", in, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeMatchTextNeverMutatesOriginal(t *testing.T) {
|
||||
inputs := []string{" привет ", "ПрИвЕт", "это вопрос?"}
|
||||
for _, in := range inputs {
|
||||
orig := in
|
||||
_ = NormalizeMatchText(in)
|
||||
if in != orig {
|
||||
t.Errorf("NormalizeMatchText mutated input: was %q, now %q", orig, in)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeMatchTextIsDeterministic(t *testing.T) {
|
||||
input := " ПрИвЕт MAVEN "
|
||||
a := NormalizeMatchText(input)
|
||||
b := NormalizeMatchText(input)
|
||||
if a != b {
|
||||
t.Errorf("NormalizeMatchText not deterministic: %q vs %q", a, b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchTextDarkDataInvariant(t *testing.T) {
|
||||
// MatchText is currently not authoritative for any routing/action result.
|
||||
// Pin this by verifying that constructing a NormalizedInput with MatchText
|
||||
// does not change the Text field.
|
||||
input := NormalizedInput{
|
||||
Text: "Привет",
|
||||
MatchText: NormalizeMatchText("Привет"),
|
||||
Source: InputSourceText,
|
||||
}
|
||||
if input.Text != "Привет" {
|
||||
t.Errorf("Text was mutated to %q", input.Text)
|
||||
}
|
||||
if input.MatchText != "привет" {
|
||||
t.Errorf("MatchText = %q, want %q", input.MatchText, "привет")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user