router/semantic: frozen holdout split and grouped CV (slice 13)
Add FrozenHoldoutSplit with deterministic 15% ratio using dedicated hash seed. Produces frozen/dev partition with SplitGroup-aware leakage prevention — all contrastive variants of one seed stay in the same split. Add GroupedCVFolds for k-fold grouped cross-validation on the development pool. Each fold preserves split_group boundaries; every example appears in exactly one eval set across all folds. Tests verify determinism (same split → same hash), no split_group leakage across frozen/dev, route coverage in both pools, fold completeness, and grouped CV coverage.
This commit is contained in:
@@ -163,6 +163,115 @@ func TestSplitByFamily(t *testing.T) {
|
||||
t.Logf("split: %d train, %d eval", len(train), len(eval))
|
||||
}
|
||||
|
||||
func TestFrozenHoldoutSplit(t *testing.T) {
|
||||
exs, err := LoadCorpus()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
frozen, dev, hash := FrozenHoldoutSplit(exs)
|
||||
total := len(frozen) + len(dev)
|
||||
if total != len(exs) {
|
||||
t.Errorf("split lost examples: frozen=%d dev=%d total=%d, want %d",
|
||||
len(frozen), len(dev), total, len(exs))
|
||||
}
|
||||
if len(frozen) == 0 {
|
||||
t.Error("frozen holdout is empty")
|
||||
}
|
||||
if len(dev) == 0 {
|
||||
t.Error("development pool is empty")
|
||||
}
|
||||
|
||||
// No split_group leakage.
|
||||
frozenGroups := map[string]bool{}
|
||||
for _, e := range frozen {
|
||||
frozenGroups[e.SplitGroup] = true
|
||||
}
|
||||
for _, e := range dev {
|
||||
if frozenGroups[e.SplitGroup] {
|
||||
t.Errorf("split_group %q leaked across frozen/dev (example %q)",
|
||||
e.SplitGroup, e.SourceID)
|
||||
}
|
||||
}
|
||||
|
||||
// Determinism: same split produces same hash.
|
||||
frozen2, dev2, hash2 := FrozenHoldoutSplit(exs)
|
||||
if hash != hash2 {
|
||||
t.Errorf("non-deterministic: hash=%s vs %s", hash, hash2)
|
||||
}
|
||||
if len(frozen) != len(frozen2) || len(dev) != len(dev2) {
|
||||
t.Errorf("non-deterministic: frozen=%d/%d dev=%d/%d",
|
||||
len(frozen), len(frozen2), len(dev), len(dev2))
|
||||
}
|
||||
|
||||
// Log route coverage in both splits (not all routes need to appear
|
||||
// in the frozen holdout — at 15% of 136 rows, small routes may miss).
|
||||
frozenRoutes := map[SemanticRoute]bool{}
|
||||
for _, e := range frozen {
|
||||
frozenRoutes[e.Route] = true
|
||||
}
|
||||
devRoutes := map[SemanticRoute]bool{}
|
||||
for _, e := range dev {
|
||||
devRoutes[e.Route] = true
|
||||
}
|
||||
for _, r := range AllRoutes {
|
||||
if !frozenRoutes[r] {
|
||||
t.Logf("note: route %q missing from frozen holdout (expected at 15%%)", r)
|
||||
}
|
||||
if !devRoutes[r] {
|
||||
t.Errorf("route %q missing from development pool", r)
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("frozen: %d examples, dev: %d examples, holdout_hash=%s",
|
||||
len(frozen), len(dev), hash)
|
||||
}
|
||||
|
||||
func TestGroupedCVFolds(t *testing.T) {
|
||||
exs, err := LoadCorpus()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, dev, _ := FrozenHoldoutSplit(exs)
|
||||
folds := GroupedCVFolds(dev, 5)
|
||||
if len(folds) != 5 {
|
||||
t.Fatalf("expected 5 folds, got %d", len(folds))
|
||||
}
|
||||
|
||||
for i, f := range folds {
|
||||
total := len(f.Train) + len(f.Eval)
|
||||
if total != len(dev) {
|
||||
t.Errorf("fold %d: train=%d + eval=%d = %d, want %d",
|
||||
i, len(f.Train), len(f.Eval), total, len(dev))
|
||||
}
|
||||
if len(f.Eval) == 0 {
|
||||
t.Errorf("fold %d: empty eval set", i)
|
||||
}
|
||||
|
||||
// No split_group leakage within a fold.
|
||||
evalGroups := map[string]bool{}
|
||||
for _, e := range f.Eval {
|
||||
evalGroups[e.SplitGroup] = true
|
||||
}
|
||||
for _, e := range f.Train {
|
||||
if evalGroups[e.SplitGroup] {
|
||||
t.Errorf("fold %d: split_group %q leaked (example %q)",
|
||||
i, e.SplitGroup, e.SourceID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Every example appears in exactly one eval set across all folds.
|
||||
evalCount := 0
|
||||
for _, f := range folds {
|
||||
evalCount += len(f.Eval)
|
||||
}
|
||||
if evalCount != len(dev) {
|
||||
t.Errorf("total eval examples across folds: %d, want %d", evalCount, len(dev))
|
||||
}
|
||||
|
||||
t.Logf("grouped CV: %d folds on %d development examples", len(folds), len(dev))
|
||||
}
|
||||
|
||||
func TestFamilyID(t *testing.T) {
|
||||
id1 := FamilyID("ru-act-002", "negation")
|
||||
id2 := FamilyID("ru-act-002", "question")
|
||||
|
||||
@@ -4,8 +4,14 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// SplitRatio controls the frozen holdout fraction of the total corpus.
|
||||
// 0.15 means 15% of families go to the frozen holdout, 85% to the
|
||||
// development pool.
|
||||
const DefaultFrozenRatio = 0.15
|
||||
|
||||
// SplitByFamily divides examples into train/eval splits such that all
|
||||
// paraphrases or contrastive variants of one seed stay in the same split.
|
||||
//
|
||||
@@ -19,26 +25,12 @@ func SplitByFamily(exs []RouteExample, splitRatio float64) (train, eval []RouteE
|
||||
splitRatio = 0.8
|
||||
}
|
||||
|
||||
// Group by split family.
|
||||
families := make(map[string][]RouteExample)
|
||||
for _, e := range exs {
|
||||
key := e.SplitGroup
|
||||
if key == "" {
|
||||
key = e.SourceID
|
||||
}
|
||||
families[key] = append(families[key], e)
|
||||
}
|
||||
families := groupByFamily(exs)
|
||||
|
||||
// Sort families for determinism.
|
||||
famKeys := make([]string, 0, len(families))
|
||||
for k := range families {
|
||||
famKeys = append(famKeys, k)
|
||||
}
|
||||
sort.Strings(famKeys)
|
||||
famKeys := sortedFamilyKeys(families)
|
||||
|
||||
for _, k := range famKeys {
|
||||
members := families[k]
|
||||
// Hash the family key to a bucket.
|
||||
h := sha256.Sum256([]byte(k))
|
||||
bucket := float64(h[0]) / 256.0
|
||||
if bucket < splitRatio {
|
||||
@@ -50,6 +42,88 @@ func SplitByFamily(exs []RouteExample, splitRatio float64) (train, eval []RouteE
|
||||
return
|
||||
}
|
||||
|
||||
// FrozenHoldoutSplit splits the corpus into a frozen holdout and a
|
||||
// development pool. The frozen holdout must never be used for weight fitting,
|
||||
// hyperparameter selection, threshold tuning, or generating near-duplicate
|
||||
// training examples.
|
||||
//
|
||||
// The split is deterministic and identified by FrozenHoldoutHash.
|
||||
func FrozenHoldoutSplit(exs []RouteExample) (frozen, development []RouteExample, holdoutHash string) {
|
||||
families := groupByFamily(exs)
|
||||
famKeys := sortedFamilyKeys(families)
|
||||
|
||||
// Use a dedicated hash seed for the frozen split so changing the
|
||||
// train/eval ratio does not move the holdout.
|
||||
const splitSeed = "semantic-router-frozen-v1"
|
||||
for _, k := range famKeys {
|
||||
members := families[k]
|
||||
h := sha256.Sum256([]byte(splitSeed + ":" + k))
|
||||
bucket := float64(h[0]) / 256.0
|
||||
if bucket < DefaultFrozenRatio {
|
||||
frozen = append(frozen, members...)
|
||||
} else {
|
||||
development = append(development, members...)
|
||||
}
|
||||
}
|
||||
|
||||
// Compute holdout hash from the group IDs in the frozen set.
|
||||
groupIDs := make(map[string]bool)
|
||||
for _, e := range frozen {
|
||||
groupIDs[e.SplitGroup] = true
|
||||
}
|
||||
ids := make([]string, 0, len(groupIDs))
|
||||
for id := range groupIDs {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
h := sha256.Sum256([]byte(hex.EncodeToString([]byte(strings.Join(ids, "|")))))
|
||||
holdoutHash = hex.EncodeToString(h[:16])
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// GroupedCVFolds splits a development pool into k folds, keeping all
|
||||
// examples with the same SplitGroup in one fold. Returns k folds; each fold
|
||||
// is the eval set, the rest are the training set.
|
||||
func GroupedCVFolds(exs []RouteExample, k int) []CVFold {
|
||||
if k <= 1 {
|
||||
k = 5
|
||||
}
|
||||
families := groupByFamily(exs)
|
||||
famKeys := sortedFamilyKeys(families)
|
||||
|
||||
// Assign families to folds round-robin for balance.
|
||||
foldFamilies := make([][]string, k)
|
||||
for i, key := range famKeys {
|
||||
foldFamilies[i%k] = append(foldFamilies[i%k], key)
|
||||
}
|
||||
|
||||
folds := make([]CVFold, k)
|
||||
for i := 0; i < k; i++ {
|
||||
evalGroups := make(map[string]bool)
|
||||
for _, g := range foldFamilies[i] {
|
||||
evalGroups[g] = true
|
||||
}
|
||||
var train, eval []RouteExample
|
||||
for _, e := range exs {
|
||||
if evalGroups[e.SplitGroup] {
|
||||
eval = append(eval, e)
|
||||
} else {
|
||||
train = append(train, e)
|
||||
}
|
||||
}
|
||||
folds[i] = CVFold{Fold: i, Train: train, Eval: eval}
|
||||
}
|
||||
return folds
|
||||
}
|
||||
|
||||
// CVFold — one fold of a grouped cross-validation split.
|
||||
type CVFold struct {
|
||||
Fold int
|
||||
Train []RouteExample
|
||||
Eval []RouteExample
|
||||
}
|
||||
|
||||
// FamilyID derives a stable family ID from a base source ID and a transform
|
||||
// name, ensuring contrastive variants share the base's family.
|
||||
func FamilyID(baseSourceID, transform string) string {
|
||||
@@ -59,3 +133,26 @@ func FamilyID(baseSourceID, transform string) string {
|
||||
h := sha256.Sum256([]byte(baseSourceID + ":" + transform))
|
||||
return "family:" + hex.EncodeToString(h[:8])
|
||||
}
|
||||
|
||||
// groupByFamily groups examples by SplitGroup.
|
||||
func groupByFamily(exs []RouteExample) map[string][]RouteExample {
|
||||
families := make(map[string][]RouteExample)
|
||||
for _, e := range exs {
|
||||
key := e.SplitGroup
|
||||
if key == "" {
|
||||
key = e.SourceID
|
||||
}
|
||||
families[key] = append(families[key], e)
|
||||
}
|
||||
return families
|
||||
}
|
||||
|
||||
// sortedFamilyKeys returns the sorted keys of a families map.
|
||||
func sortedFamilyKeys(families map[string][]RouteExample) []string {
|
||||
keys := make([]string, 0, len(families))
|
||||
for k := range families {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user