router/semantic: contrastive safety transforms and split-by-family (slice 12)

Six deterministic transforms (negation, question, reported speech,
quotation, hypothetical, capability question) applied to action-route
seeds. Each transform determines the expected class explicitly — no
model guessing labels. SplitByFamily uses hash-based bucketing to keep
paraphrases in the same split.

internal/router/semantic/split.go

@@ -0,0 +1,61 @@
+package semantic
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"sort"
+)
+
+// SplitByFamily divides examples into train/eval splits such that all
+// paraphrases or contrastive variants of one seed stay in the same split.
+//
+// The algorithm is deterministic: each SplitGroup gets a hash, and the hash
+// bucket determines the split. This prevents the failure mode where
+// "выключи свет" lands in train and "пожалуйста выключи свет" lands in eval.
+//
+// splitRatio controls the train fraction (0.0–1.0). 0.8 means 80% train.
+func SplitByFamily(exs []RouteExample, splitRatio float64) (train, eval []RouteExample) {
+	if splitRatio <= 0 || splitRatio >= 1 {
+		splitRatio = 0.8
+	}
+
+	// Group by split family.
+	families := make(map[string][]RouteExample)
+	for _, e := range exs {
+		key := e.SplitGroup
+		if key == "" {
+			key = e.SourceID
+		}
+		families[key] = append(families[key], e)
+	}
+
+	// Sort families for determinism.
+	famKeys := make([]string, 0, len(families))
+	for k := range families {
+		famKeys = append(famKeys, k)
+	}
+	sort.Strings(famKeys)
+
+	for _, k := range famKeys {
+		members := families[k]
+		// Hash the family key to a bucket.
+		h := sha256.Sum256([]byte(k))
+		bucket := float64(h[0]) / 256.0
+		if bucket < splitRatio {
+			train = append(train, members...)
+		} else {
+			eval = append(eval, members...)
+		}
+	}
+	return
+}
+
+// FamilyID derives a stable family ID from a base source ID and a transform
+// name, ensuring contrastive variants share the base's family.
+func FamilyID(baseSourceID, transform string) string {
+	if transform == "" {
+		return baseSourceID
+	}
+	h := sha256.Sum256([]byte(baseSourceID + ":" + transform))
+	return "family:" + hex.EncodeToString(h[:8])
+}

internal/router/semantic/transform.go

@@ -0,0 +1,226 @@
+package semantic
+
+import "fmt"
+
+// ContrastivePair — one base utterance and a derived transformation that
+// should land in a different (or same) coarse route. The transform tag
+// identifies the operation so the eval can report which transformations are
+// easy vs hard.
+type ContrastivePair struct {
+	BaseID    string        `json:"base_id"`
+	BaseText  string        `json:"base_text"`
+	BaseRoute SemanticRoute `json:"base_route"`
+	Transform string        `json:"transform"`
+	Text      string        `json:"text"`
+	Route     SemanticRoute `json:"route"`
+}
+
+// ContrastiveTransform — a function that takes a base utterance and returns
+// a list of (text, expected_route) pairs. The mapping is deterministic and
+// does not consult any model.
+type ContrastiveTransform struct {
+	Name string
+	Fn   func(baseID, baseText string, baseRoute SemanticRoute) []ContrastivePair
+}
+
+// StandardTransforms is the ordered set of contrastive safety transformations
+// applied to the action-route seeds. Each base utterance gets all transforms;
+// the expected route depends on the transformation semantics and the current
+// routing contract.
+var StandardTransforms = []ContrastiveTransform{
+	{Name: "negation", Fn: negationTransform},
+	{Name: "question", Fn: questionTransform},
+	{Name: "reported_speech", Fn: reportedSpeechTransform},
+	{Name: "quotation", Fn: quotationTransform},
+	{Name: "hypothetical", Fn: hypotheticalTransform},
+	{Name: "capability_question", Fn: capabilityQuestionTransform},
+}
+
+// negationTransform — turns an imperative into a negated one. A negated
+// command is not an executable action; current semantics route it to
+// uncertain.
+//
+//	"выключи свет" → "не выключай свет" → uncertain
+func negationTransform(baseID, baseText string, baseRoute SemanticRoute) []ContrastivePair {
+	// Only transform action-route examples: other routes are unaffected.
+	if baseRoute != RouteAction {
+		return nil
+	}
+	ruNegations := []struct{ prefix, suffix string }{
+		{"не ", ""},
+		{"ни ", ""},
+	}
+	var pairs []ContrastivePair
+	for _, n := range ruNegations {
+		text := n.prefix + baseText + n.suffix
+		pairs = append(pairs, ContrastivePair{
+			BaseID:    baseID,
+			BaseText:  baseText,
+			BaseRoute: baseRoute,
+			Transform: "negation",
+			Text:      text,
+			Route:     RouteUncertain,
+		})
+	}
+	return pairs
+}
+
+// questionTransform — turns a statement into a question. A question about an
+// action is not the action itself; it becomes knowledge.
+//
+//	"выключи свет" → "ты выключила свет?" → knowledge
+func questionTransform(baseID, baseText string, baseRoute SemanticRoute) []ContrastivePair {
+	if baseRoute != RouteAction {
+		return nil
+	}
+	// Russian question forms for actions
+	suffixes := []string{
+		"?", // direct question
+		", верно?", // tag question
+	}
+	var pairs []ContrastivePair
+	for _, s := range suffixes {
+		text := baseText + s
+		pairs = append(pairs, ContrastivePair{
+			BaseID:    baseID,
+			BaseText:  baseText,
+			BaseRoute: baseRoute,
+			Transform: "question",
+			Text:      text,
+			Route:     RouteKnowledge,
+		})
+	}
+	return pairs
+}
+
+// reportedSpeechTransform — wraps the utterance in reported speech. An action
+// reported as speech is not an action.
+//
+//	"выключи свет" → "он сказал выключи свет" → uncertain
+func reportedSpeechTransform(baseID, baseText string, baseRoute SemanticRoute) []ContrastivePair {
+	if baseRoute != RouteAction {
+		return nil
+	}
+	prefixes := []string{
+		"он сказал: ",
+		"она сказала: ",
+		"он сказал «",
+	}
+	suffixes := []string{
+		"",
+		"",
+		"»",
+	}
+	var pairs []ContrastivePair
+	for i, p := range prefixes {
+		text := p + baseText + suffixes[i]
+		pairs = append(pairs, ContrastivePair{
+			BaseID:    baseID,
+			BaseText:  baseText,
+			BaseRoute: baseRoute,
+			Transform: "reported_speech",
+			Text:      text,
+			Route:     RouteUncertain,
+		})
+	}
+	return pairs
+}
+
+// quotationTransform — mentions the utterance as a phrase/quote, not as a
+// command. Quoting an action is not doing it.
+//
+//	"выключи свет" → "фраза «выключи свет»" → uncertain
+func quotationTransform(baseID, baseText string, baseRoute SemanticRoute) []ContrastivePair {
+	if baseRoute != RouteAction {
+		return nil
+	}
+	patterns := []struct{ pre, post string }{
+		{"фраза «", "»"},
+		{"«", "»"},
+		{"цитата: \"", "\""},
+	}
+	var pairs []ContrastivePair
+	for _, p := range patterns {
+		text := p.pre + baseText + p.post
+		pairs = append(pairs, ContrastivePair{
+			BaseID:    baseID,
+			BaseText:  baseText,
+			BaseRoute: baseRoute,
+			Transform: "quotation",
+			Text:      text,
+			Route:     RouteUncertain,
+		})
+	}
+	return pairs
+}
+
+// hypotheticalTransform — puts the action in a hypothetical frame. An
+// "if..." clause is not an executable command.
+//
+//	"выключи свет" → "если выключить свет..." → uncertain
+func hypotheticalTransform(baseID, baseText string, baseRoute SemanticRoute) []ContrastivePair {
+	if baseRoute != RouteAction {
+		return nil
+	}
+	frames := []struct{ pre, post string }{
+		{"если ", "..."},
+		{"когда ", ", будет проще"},
+		{"если бы я сказал: ", ", что бы ты сделала?"},
+	}
+	var pairs []ContrastivePair
+	for _, f := range frames {
+		text := f.pre + baseText + f.post
+		pairs = append(pairs, ContrastivePair{
+			BaseID:    baseID,
+			BaseText:  baseText,
+			BaseRoute: baseRoute,
+			Transform: "hypothetical",
+			Text:      text,
+			Route:     RouteUncertain,
+		})
+	}
+	return pairs
+}
+
+// capabilityQuestionTransform — asks whether the system CAN do the action.
+// A capability question is knowledge, not a direct executable command.
+//
+//	"выключи свет" → "ты можешь выключить свет?" → knowledge
+func capabilityQuestionTransform(baseID, baseText string, baseRoute SemanticRoute) []ContrastivePair {
+	if baseRoute != RouteAction {
+		return nil
+	}
+	templates := []string{
+		"ты можешь %s?",
+		"умеешь ли %s?",
+		"способен ли ты %s?",
+	}
+	// Extract the verb phrase for templates that need infinitive.
+	// For Russian, we use the base text as-is since the template
+	// handles the grammar.
+	var pairs []ContrastivePair
+	for _, t := range templates {
+		text := fmt.Sprintf(t, baseText)
+		pairs = append(pairs, ContrastivePair{
+			BaseID:    baseID,
+			BaseText:  baseText,
+			BaseRoute: baseRoute,
+			Transform: "capability_question",
+			Text:      text,
+			Route:     RouteKnowledge,
+		})
+	}
+	return pairs
+}
+
+// GenerateContrastivePairs applies all standard transforms to a slice of
+// base examples and returns the full set of contrastive pairs.
+func GenerateContrastivePairs(bases []RouteExample) []ContrastivePair {
+	var all []ContrastivePair
+	for _, b := range bases {
+		for _, t := range StandardTransforms {
+			all = append(all, t.Fn(b.SourceID, b.Text, b.Route)...)
+		}
+	}
+	return all
+}