router/semantic: frozen holdout split and grouped CV (slice 13)
Add FrozenHoldoutSplit with deterministic 15% ratio using dedicated hash seed. Produces frozen/dev partition with SplitGroup-aware leakage prevention — all contrastive variants of one seed stay in the same split. Add GroupedCVFolds for k-fold grouped cross-validation on the development pool. Each fold preserves split_group boundaries; every example appears in exactly one eval set across all folds. Tests verify determinism (same split → same hash), no split_group leakage across frozen/dev, route coverage in both pools, fold completeness, and grouped CV coverage.
This commit is contained in:
@@ -163,6 +163,115 @@ func TestSplitByFamily(t *testing.T) {
|
||||
t.Logf("split: %d train, %d eval", len(train), len(eval))
|
||||
}
|
||||
|
||||
func TestFrozenHoldoutSplit(t *testing.T) {
|
||||
exs, err := LoadCorpus()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
frozen, dev, hash := FrozenHoldoutSplit(exs)
|
||||
total := len(frozen) + len(dev)
|
||||
if total != len(exs) {
|
||||
t.Errorf("split lost examples: frozen=%d dev=%d total=%d, want %d",
|
||||
len(frozen), len(dev), total, len(exs))
|
||||
}
|
||||
if len(frozen) == 0 {
|
||||
t.Error("frozen holdout is empty")
|
||||
}
|
||||
if len(dev) == 0 {
|
||||
t.Error("development pool is empty")
|
||||
}
|
||||
|
||||
// No split_group leakage.
|
||||
frozenGroups := map[string]bool{}
|
||||
for _, e := range frozen {
|
||||
frozenGroups[e.SplitGroup] = true
|
||||
}
|
||||
for _, e := range dev {
|
||||
if frozenGroups[e.SplitGroup] {
|
||||
t.Errorf("split_group %q leaked across frozen/dev (example %q)",
|
||||
e.SplitGroup, e.SourceID)
|
||||
}
|
||||
}
|
||||
|
||||
// Determinism: same split produces same hash.
|
||||
frozen2, dev2, hash2 := FrozenHoldoutSplit(exs)
|
||||
if hash != hash2 {
|
||||
t.Errorf("non-deterministic: hash=%s vs %s", hash, hash2)
|
||||
}
|
||||
if len(frozen) != len(frozen2) || len(dev) != len(dev2) {
|
||||
t.Errorf("non-deterministic: frozen=%d/%d dev=%d/%d",
|
||||
len(frozen), len(frozen2), len(dev), len(dev2))
|
||||
}
|
||||
|
||||
// Log route coverage in both splits (not all routes need to appear
|
||||
// in the frozen holdout — at 15% of 136 rows, small routes may miss).
|
||||
frozenRoutes := map[SemanticRoute]bool{}
|
||||
for _, e := range frozen {
|
||||
frozenRoutes[e.Route] = true
|
||||
}
|
||||
devRoutes := map[SemanticRoute]bool{}
|
||||
for _, e := range dev {
|
||||
devRoutes[e.Route] = true
|
||||
}
|
||||
for _, r := range AllRoutes {
|
||||
if !frozenRoutes[r] {
|
||||
t.Logf("note: route %q missing from frozen holdout (expected at 15%%)", r)
|
||||
}
|
||||
if !devRoutes[r] {
|
||||
t.Errorf("route %q missing from development pool", r)
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("frozen: %d examples, dev: %d examples, holdout_hash=%s",
|
||||
len(frozen), len(dev), hash)
|
||||
}
|
||||
|
||||
func TestGroupedCVFolds(t *testing.T) {
|
||||
exs, err := LoadCorpus()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, dev, _ := FrozenHoldoutSplit(exs)
|
||||
folds := GroupedCVFolds(dev, 5)
|
||||
if len(folds) != 5 {
|
||||
t.Fatalf("expected 5 folds, got %d", len(folds))
|
||||
}
|
||||
|
||||
for i, f := range folds {
|
||||
total := len(f.Train) + len(f.Eval)
|
||||
if total != len(dev) {
|
||||
t.Errorf("fold %d: train=%d + eval=%d = %d, want %d",
|
||||
i, len(f.Train), len(f.Eval), total, len(dev))
|
||||
}
|
||||
if len(f.Eval) == 0 {
|
||||
t.Errorf("fold %d: empty eval set", i)
|
||||
}
|
||||
|
||||
// No split_group leakage within a fold.
|
||||
evalGroups := map[string]bool{}
|
||||
for _, e := range f.Eval {
|
||||
evalGroups[e.SplitGroup] = true
|
||||
}
|
||||
for _, e := range f.Train {
|
||||
if evalGroups[e.SplitGroup] {
|
||||
t.Errorf("fold %d: split_group %q leaked (example %q)",
|
||||
i, e.SplitGroup, e.SourceID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Every example appears in exactly one eval set across all folds.
|
||||
evalCount := 0
|
||||
for _, f := range folds {
|
||||
evalCount += len(f.Eval)
|
||||
}
|
||||
if evalCount != len(dev) {
|
||||
t.Errorf("total eval examples across folds: %d, want %d", evalCount, len(dev))
|
||||
}
|
||||
|
||||
t.Logf("grouped CV: %d folds on %d development examples", len(folds), len(dev))
|
||||
}
|
||||
|
||||
func TestFamilyID(t *testing.T) {
|
||||
id1 := FamilyID("ru-act-002", "negation")
|
||||
id2 := FamilyID("ru-act-002", "question")
|
||||
|
||||
Reference in New Issue
Block a user