router: add MatchText to NormalizedInput and NormalizeMatchText function (slice 9a)
Add MatchText field to NormalizedInput — a lossy lexical matching view derived from ingress text: TrimSpace → NFKC → lowercase → collapse Unicode whitespace. Does NOT fold ё→е, strip punctuation, strip wake words, rewrite numbers, or invoke morphology. Both ingress sites (voice + text) construct MatchText at entry. No existing consumer reads MatchText yet — it is dark data for future opt-in migration. Vikunja: #725
This commit is contained in:
+2
-2
@@ -211,7 +211,7 @@ func (h *reactiveHandler) HandlePushToTalk(ctx context.Context, req voice.PushTo
|
||||
|
||||
// 2-5. the shared turn pipeline (confirm → clarify → route → dialogue →
|
||||
// action → replier), identical to the text path.
|
||||
replyText := h.runTurn(ctx, router.NormalizedInput{Text: text, Source: sourceVoice})
|
||||
replyText := h.runTurn(ctx, router.NormalizedInput{Text: text, MatchText: router.NormalizeMatchText(text), Source: sourceVoice})
|
||||
|
||||
// 6. tts — synthesise the reply text; return to the voice server which
|
||||
// ships it back on the conn.
|
||||
@@ -244,7 +244,7 @@ func (h *reactiveHandler) upgradeAPI(api ipc.CoreAPI) {
|
||||
// HandlePushToTalk so text channels share the same routing logic.
|
||||
func (h *reactiveHandler) handleText(ctx context.Context, conversation, text string) string {
|
||||
log.Printf("voice: handleText: %q", text)
|
||||
return h.runTurn(withDialogueID(ctx, dialogueIDFor(sourceText, conversation)), router.NormalizedInput{Text: text, Source: sourceText})
|
||||
return h.runTurn(withDialogueID(ctx, dialogueIDFor(sourceText, conversation)), router.NormalizedInput{Text: text, MatchText: router.NormalizeMatchText(text), Source: sourceText})
|
||||
}
|
||||
|
||||
// turnSource is a local alias for router.InputSource, kept so the daemon code
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
// NormalizeMatchText derives a lossy lexical matching view from raw ingress text.
|
||||
// The result is intended ONLY for consumers that explicitly opt into
|
||||
// case-insensitive, whitespace-collapsed matching. It must not be used for
|
||||
// user-visible text, entity identity, quoted content, free-form arguments,
|
||||
// or persisted utterances without proving the transformation is safe for
|
||||
// that consumer.
|
||||
//
|
||||
// Pipeline: TrimSpace → Unicode NFKC → lowercase → collapse Unicode whitespace.
|
||||
// Does NOT fold ё→е, strip punctuation, strip wake words, rewrite numbers,
|
||||
// or invoke morphology.
|
||||
func NormalizeMatchText(in string) string {
|
||||
out := strings.TrimSpace(in)
|
||||
out = norm.NFKC.String(out)
|
||||
out = strings.ToLower(out)
|
||||
// Collapse runs of Unicode whitespace (including non-breaking space,
|
||||
// thin space, ideographic space, etc.) to a single ASCII space.
|
||||
out = strings.Join(strings.FieldsFunc(out, func(r rune) bool {
|
||||
return unicode.IsSpace(r)
|
||||
}), " ")
|
||||
return out
|
||||
}
|
||||
|
||||
|
||||
@@ -92,10 +92,12 @@ const (
|
||||
)
|
||||
|
||||
// NormalizedInput — the typed ingress boundary for a turn. Text is the raw
|
||||
// utterance after STT (voice) or as typed (text). Source identifies the
|
||||
// channel. This slice performs no new linguistic normalization: text and voice
|
||||
// paths continue to converge onto the same turn path as they did before.
|
||||
// utterance after STT (voice) or as typed (text). MatchText is a lossy
|
||||
// lexical matching view derived from Text: whitespace-collapsed, NFKC-
|
||||
// normalized, lowercased. Consumers must opt into MatchText individually;
|
||||
// nothing reads it by default in this slice. Source identifies the channel.
|
||||
type NormalizedInput struct {
|
||||
Text string
|
||||
Source InputSource
|
||||
Text string // original ingress text — byte-for-byte the value current consumers receive
|
||||
MatchText string // derived lossy representation for case-insensitive lexical matching
|
||||
Source InputSource
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user