From d63619bd0ee1fea5f5f9badc7481d26e22e2f353 Mon Sep 17 00:00:00 2001 From: claude Date: Mon, 7 Sep 2026 02:08:30 +0400 Subject: [PATCH] router/semantic: frozen holdout split and grouped CV (slice 13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add FrozenHoldoutSplit with deterministic 15% ratio using dedicated hash seed. Produces frozen/dev partition with SplitGroup-aware leakage prevention — all contrastive variants of one seed stay in the same split. Add GroupedCVFolds for k-fold grouped cross-validation on the development pool. Each fold preserves split_group boundaries; every example appears in exactly one eval set across all folds. Tests verify determinism (same split → same hash), no split_group leakage across frozen/dev, route coverage in both pools, fold completeness, and grouped CV coverage. --- internal/router/semantic/contract_test.go | 109 ++++++++++++++++++ internal/router/semantic/split.go | 129 +++++++++++++++++++--- 2 files changed, 222 insertions(+), 16 deletions(-) diff --git a/internal/router/semantic/contract_test.go b/internal/router/semantic/contract_test.go index ac3cb92..2754fc4 100644 --- a/internal/router/semantic/contract_test.go +++ b/internal/router/semantic/contract_test.go @@ -163,6 +163,115 @@ func TestSplitByFamily(t *testing.T) { t.Logf("split: %d train, %d eval", len(train), len(eval)) } +func TestFrozenHoldoutSplit(t *testing.T) { + exs, err := LoadCorpus() + if err != nil { + t.Fatal(err) + } + frozen, dev, hash := FrozenHoldoutSplit(exs) + total := len(frozen) + len(dev) + if total != len(exs) { + t.Errorf("split lost examples: frozen=%d dev=%d total=%d, want %d", + len(frozen), len(dev), total, len(exs)) + } + if len(frozen) == 0 { + t.Error("frozen holdout is empty") + } + if len(dev) == 0 { + t.Error("development pool is empty") + } + + // No split_group leakage. + frozenGroups := map[string]bool{} + for _, e := range frozen { + frozenGroups[e.SplitGroup] = true + } + for _, e := range dev { + if frozenGroups[e.SplitGroup] { + t.Errorf("split_group %q leaked across frozen/dev (example %q)", + e.SplitGroup, e.SourceID) + } + } + + // Determinism: same split produces same hash. + frozen2, dev2, hash2 := FrozenHoldoutSplit(exs) + if hash != hash2 { + t.Errorf("non-deterministic: hash=%s vs %s", hash, hash2) + } + if len(frozen) != len(frozen2) || len(dev) != len(dev2) { + t.Errorf("non-deterministic: frozen=%d/%d dev=%d/%d", + len(frozen), len(frozen2), len(dev), len(dev2)) + } + + // Log route coverage in both splits (not all routes need to appear + // in the frozen holdout — at 15% of 136 rows, small routes may miss). + frozenRoutes := map[SemanticRoute]bool{} + for _, e := range frozen { + frozenRoutes[e.Route] = true + } + devRoutes := map[SemanticRoute]bool{} + for _, e := range dev { + devRoutes[e.Route] = true + } + for _, r := range AllRoutes { + if !frozenRoutes[r] { + t.Logf("note: route %q missing from frozen holdout (expected at 15%%)", r) + } + if !devRoutes[r] { + t.Errorf("route %q missing from development pool", r) + } + } + + t.Logf("frozen: %d examples, dev: %d examples, holdout_hash=%s", + len(frozen), len(dev), hash) +} + +func TestGroupedCVFolds(t *testing.T) { + exs, err := LoadCorpus() + if err != nil { + t.Fatal(err) + } + _, dev, _ := FrozenHoldoutSplit(exs) + folds := GroupedCVFolds(dev, 5) + if len(folds) != 5 { + t.Fatalf("expected 5 folds, got %d", len(folds)) + } + + for i, f := range folds { + total := len(f.Train) + len(f.Eval) + if total != len(dev) { + t.Errorf("fold %d: train=%d + eval=%d = %d, want %d", + i, len(f.Train), len(f.Eval), total, len(dev)) + } + if len(f.Eval) == 0 { + t.Errorf("fold %d: empty eval set", i) + } + + // No split_group leakage within a fold. + evalGroups := map[string]bool{} + for _, e := range f.Eval { + evalGroups[e.SplitGroup] = true + } + for _, e := range f.Train { + if evalGroups[e.SplitGroup] { + t.Errorf("fold %d: split_group %q leaked (example %q)", + i, e.SplitGroup, e.SourceID) + } + } + } + + // Every example appears in exactly one eval set across all folds. + evalCount := 0 + for _, f := range folds { + evalCount += len(f.Eval) + } + if evalCount != len(dev) { + t.Errorf("total eval examples across folds: %d, want %d", evalCount, len(dev)) + } + + t.Logf("grouped CV: %d folds on %d development examples", len(folds), len(dev)) +} + func TestFamilyID(t *testing.T) { id1 := FamilyID("ru-act-002", "negation") id2 := FamilyID("ru-act-002", "question") diff --git a/internal/router/semantic/split.go b/internal/router/semantic/split.go index f7d1a87..fac46be 100644 --- a/internal/router/semantic/split.go +++ b/internal/router/semantic/split.go @@ -4,8 +4,14 @@ import ( "crypto/sha256" "encoding/hex" "sort" + "strings" ) +// SplitRatio controls the frozen holdout fraction of the total corpus. +// 0.15 means 15% of families go to the frozen holdout, 85% to the +// development pool. +const DefaultFrozenRatio = 0.15 + // SplitByFamily divides examples into train/eval splits such that all // paraphrases or contrastive variants of one seed stay in the same split. // @@ -19,26 +25,12 @@ func SplitByFamily(exs []RouteExample, splitRatio float64) (train, eval []RouteE splitRatio = 0.8 } - // Group by split family. - families := make(map[string][]RouteExample) - for _, e := range exs { - key := e.SplitGroup - if key == "" { - key = e.SourceID - } - families[key] = append(families[key], e) - } + families := groupByFamily(exs) - // Sort families for determinism. - famKeys := make([]string, 0, len(families)) - for k := range families { - famKeys = append(famKeys, k) - } - sort.Strings(famKeys) + famKeys := sortedFamilyKeys(families) for _, k := range famKeys { members := families[k] - // Hash the family key to a bucket. h := sha256.Sum256([]byte(k)) bucket := float64(h[0]) / 256.0 if bucket < splitRatio { @@ -50,6 +42,88 @@ func SplitByFamily(exs []RouteExample, splitRatio float64) (train, eval []RouteE return } +// FrozenHoldoutSplit splits the corpus into a frozen holdout and a +// development pool. The frozen holdout must never be used for weight fitting, +// hyperparameter selection, threshold tuning, or generating near-duplicate +// training examples. +// +// The split is deterministic and identified by FrozenHoldoutHash. +func FrozenHoldoutSplit(exs []RouteExample) (frozen, development []RouteExample, holdoutHash string) { + families := groupByFamily(exs) + famKeys := sortedFamilyKeys(families) + + // Use a dedicated hash seed for the frozen split so changing the + // train/eval ratio does not move the holdout. + const splitSeed = "semantic-router-frozen-v1" + for _, k := range famKeys { + members := families[k] + h := sha256.Sum256([]byte(splitSeed + ":" + k)) + bucket := float64(h[0]) / 256.0 + if bucket < DefaultFrozenRatio { + frozen = append(frozen, members...) + } else { + development = append(development, members...) + } + } + + // Compute holdout hash from the group IDs in the frozen set. + groupIDs := make(map[string]bool) + for _, e := range frozen { + groupIDs[e.SplitGroup] = true + } + ids := make([]string, 0, len(groupIDs)) + for id := range groupIDs { + ids = append(ids, id) + } + sort.Strings(ids) + h := sha256.Sum256([]byte(hex.EncodeToString([]byte(strings.Join(ids, "|"))))) + holdoutHash = hex.EncodeToString(h[:16]) + + return +} + +// GroupedCVFolds splits a development pool into k folds, keeping all +// examples with the same SplitGroup in one fold. Returns k folds; each fold +// is the eval set, the rest are the training set. +func GroupedCVFolds(exs []RouteExample, k int) []CVFold { + if k <= 1 { + k = 5 + } + families := groupByFamily(exs) + famKeys := sortedFamilyKeys(families) + + // Assign families to folds round-robin for balance. + foldFamilies := make([][]string, k) + for i, key := range famKeys { + foldFamilies[i%k] = append(foldFamilies[i%k], key) + } + + folds := make([]CVFold, k) + for i := 0; i < k; i++ { + evalGroups := make(map[string]bool) + for _, g := range foldFamilies[i] { + evalGroups[g] = true + } + var train, eval []RouteExample + for _, e := range exs { + if evalGroups[e.SplitGroup] { + eval = append(eval, e) + } else { + train = append(train, e) + } + } + folds[i] = CVFold{Fold: i, Train: train, Eval: eval} + } + return folds +} + +// CVFold — one fold of a grouped cross-validation split. +type CVFold struct { + Fold int + Train []RouteExample + Eval []RouteExample +} + // FamilyID derives a stable family ID from a base source ID and a transform // name, ensuring contrastive variants share the base's family. func FamilyID(baseSourceID, transform string) string { @@ -59,3 +133,26 @@ func FamilyID(baseSourceID, transform string) string { h := sha256.Sum256([]byte(baseSourceID + ":" + transform)) return "family:" + hex.EncodeToString(h[:8]) } + +// groupByFamily groups examples by SplitGroup. +func groupByFamily(exs []RouteExample) map[string][]RouteExample { + families := make(map[string][]RouteExample) + for _, e := range exs { + key := e.SplitGroup + if key == "" { + key = e.SourceID + } + families[key] = append(families[key], e) + } + return families +} + +// sortedFamilyKeys returns the sorted keys of a families map. +func sortedFamilyKeys(families map[string][]RouteExample) []string { + keys := make([]string, 0, len(families)) + for k := range families { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +}