router/semantic: frozen holdout split and grouped CV (slice 13)

Add FrozenHoldoutSplit with deterministic 15% ratio using dedicated hash
seed. Produces frozen/dev partition with SplitGroup-aware leakage
prevention — all contrastive variants of one seed stay in the same split.

Add GroupedCVFolds for k-fold grouped cross-validation on the development
pool. Each fold preserves split_group boundaries; every example appears
in exactly one eval set across all folds.

Tests verify determinism (same split → same hash), no split_group leakage
across frozen/dev, route coverage in both pools, fold completeness, and
grouped CV coverage.
This commit is contained in:
2026-09-07 02:08:30 +04:00
parent fb9b719f0b
commit d63619bd0e
2 changed files with 222 additions and 16 deletions
+109
View File
@@ -163,6 +163,115 @@ func TestSplitByFamily(t *testing.T) {
t.Logf("split: %d train, %d eval", len(train), len(eval))
}
func TestFrozenHoldoutSplit(t *testing.T) {
exs, err := LoadCorpus()
if err != nil {
t.Fatal(err)
}
frozen, dev, hash := FrozenHoldoutSplit(exs)
total := len(frozen) + len(dev)
if total != len(exs) {
t.Errorf("split lost examples: frozen=%d dev=%d total=%d, want %d",
len(frozen), len(dev), total, len(exs))
}
if len(frozen) == 0 {
t.Error("frozen holdout is empty")
}
if len(dev) == 0 {
t.Error("development pool is empty")
}
// No split_group leakage.
frozenGroups := map[string]bool{}
for _, e := range frozen {
frozenGroups[e.SplitGroup] = true
}
for _, e := range dev {
if frozenGroups[e.SplitGroup] {
t.Errorf("split_group %q leaked across frozen/dev (example %q)",
e.SplitGroup, e.SourceID)
}
}
// Determinism: same split produces same hash.
frozen2, dev2, hash2 := FrozenHoldoutSplit(exs)
if hash != hash2 {
t.Errorf("non-deterministic: hash=%s vs %s", hash, hash2)
}
if len(frozen) != len(frozen2) || len(dev) != len(dev2) {
t.Errorf("non-deterministic: frozen=%d/%d dev=%d/%d",
len(frozen), len(frozen2), len(dev), len(dev2))
}
// Log route coverage in both splits (not all routes need to appear
// in the frozen holdout — at 15% of 136 rows, small routes may miss).
frozenRoutes := map[SemanticRoute]bool{}
for _, e := range frozen {
frozenRoutes[e.Route] = true
}
devRoutes := map[SemanticRoute]bool{}
for _, e := range dev {
devRoutes[e.Route] = true
}
for _, r := range AllRoutes {
if !frozenRoutes[r] {
t.Logf("note: route %q missing from frozen holdout (expected at 15%%)", r)
}
if !devRoutes[r] {
t.Errorf("route %q missing from development pool", r)
}
}
t.Logf("frozen: %d examples, dev: %d examples, holdout_hash=%s",
len(frozen), len(dev), hash)
}
func TestGroupedCVFolds(t *testing.T) {
exs, err := LoadCorpus()
if err != nil {
t.Fatal(err)
}
_, dev, _ := FrozenHoldoutSplit(exs)
folds := GroupedCVFolds(dev, 5)
if len(folds) != 5 {
t.Fatalf("expected 5 folds, got %d", len(folds))
}
for i, f := range folds {
total := len(f.Train) + len(f.Eval)
if total != len(dev) {
t.Errorf("fold %d: train=%d + eval=%d = %d, want %d",
i, len(f.Train), len(f.Eval), total, len(dev))
}
if len(f.Eval) == 0 {
t.Errorf("fold %d: empty eval set", i)
}
// No split_group leakage within a fold.
evalGroups := map[string]bool{}
for _, e := range f.Eval {
evalGroups[e.SplitGroup] = true
}
for _, e := range f.Train {
if evalGroups[e.SplitGroup] {
t.Errorf("fold %d: split_group %q leaked (example %q)",
i, e.SplitGroup, e.SourceID)
}
}
}
// Every example appears in exactly one eval set across all folds.
evalCount := 0
for _, f := range folds {
evalCount += len(f.Eval)
}
if evalCount != len(dev) {
t.Errorf("total eval examples across folds: %d, want %d", evalCount, len(dev))
}
t.Logf("grouped CV: %d folds on %d development examples", len(folds), len(dev))
}
func TestFamilyID(t *testing.T) {
id1 := FamilyID("ru-act-002", "negation")
id2 := FamilyID("ru-act-002", "question")