router/semantic: baseline types, test helpers, leakage test (slice 13)
Baseline types: LegacyRouter interface (satisfied by *router.Router), LegacyCase with PrerouteConsumed flag for command-prohibition detection, LegacyReport with fast-path/residual/router-residual/pre-route breakdowns, ContrastFamilyReport for per-transform-family scoring. Test helpers: buildSeededClassifier (hash embedder seeded from models/seeds/*.txt), actVerbList, seed file loading. TestContrastFamiliesShareSplitGroup verifies that all contrastive variants of one base seed share exactly one SplitGroup, preventing train/eval leakage across the contrast family split.
This commit is contained in:
@@ -0,0 +1,67 @@
|
||||
package semantic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/router"
|
||||
)
|
||||
|
||||
// LegacyRouter — the interface for scoring the current routing machinery.
|
||||
// *router.Router satisfies this directly. Tests can substitute a stub.
|
||||
type LegacyRouter interface {
|
||||
Route(ctx context.Context, input router.NormalizedInput, now time.Time) (router.Decision, error)
|
||||
}
|
||||
|
||||
// LegacyCase — one scored row from the legacy baseline.
|
||||
type LegacyCase struct {
|
||||
Example RouteExample
|
||||
Decision router.Decision
|
||||
Predicted SemanticRoute
|
||||
Agree bool
|
||||
FastPathHit bool
|
||||
// PrerouteConsumed is true when a pre-route resolver (e.g.
|
||||
// command-prohibition grammar) consumed this turn before the general
|
||||
// cascade. These cases never reach the classifier or the LLM.
|
||||
PrerouteConsumed bool
|
||||
Error error
|
||||
}
|
||||
|
||||
// LegacyReport — aggregate metrics from running the real router against the
|
||||
// coarse-route corpus.
|
||||
type LegacyReport struct {
|
||||
Stats CorpusStats
|
||||
Total int
|
||||
Passed int
|
||||
ByRoute map[SemanticRoute]RouteMetrics
|
||||
// FalseAction — cases expected non-action but the router predicted action.
|
||||
FalseAction int
|
||||
FalseActionRate float64
|
||||
// FalseActionCases — the individual cases, for inspection.
|
||||
FalseActionCases []LegacyCase
|
||||
// Confusion[want][got]
|
||||
Confusion map[SemanticRoute]map[SemanticRoute]int
|
||||
// Fast-path breakdown
|
||||
FastPathTotal int
|
||||
FastPathPassed int
|
||||
ResidualTotal int
|
||||
ResidualPassed int
|
||||
// Router-residual breakdown: cases that reach the general cascade
|
||||
// (not consumed by pre-route resolvers like command-prohibition).
|
||||
RouterResidualTotal int
|
||||
RouterResidualPassed int
|
||||
PreRouteTotal int // cases consumed by command-prohibition grammar
|
||||
PreRoutePassed int
|
||||
// Cases by individual outcome
|
||||
Cases []LegacyCase
|
||||
}
|
||||
|
||||
// ContrastFamilyReport — per-transform-family breakdown.
|
||||
type ContrastFamilyReport struct {
|
||||
Transform string
|
||||
Examples int
|
||||
Correct int
|
||||
FalseAction int // predicted action when expected non-action
|
||||
OtherErrors int
|
||||
Cases []LegacyCase
|
||||
}
|
||||
@@ -287,3 +287,53 @@ func TestFamilyID(t *testing.T) {
|
||||
t.Error("different base IDs produced same FamilyID")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContrastFamiliesShareSplitGroup(t *testing.T) {
|
||||
exs, err := LoadCorpus()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Group contrastive examples by their base source_id.
|
||||
baseGroups := map[string]map[string]bool{} // source_id → {split_group: true}
|
||||
for _, e := range exs {
|
||||
if e.Source != "contrastive" {
|
||||
continue
|
||||
}
|
||||
if baseGroups[e.SourceID] == nil {
|
||||
baseGroups[e.SourceID] = map[string]bool{}
|
||||
}
|
||||
baseGroups[e.SourceID][e.SplitGroup] = true
|
||||
}
|
||||
|
||||
// Every base seed's contrastive variants must share exactly one
|
||||
// split_group (the base seed's own group).
|
||||
for sourceID, groups := range baseGroups {
|
||||
if len(groups) != 1 {
|
||||
t.Errorf("base %q has contrastive variants in %d split groups: %v",
|
||||
sourceID, len(groups), groups)
|
||||
}
|
||||
}
|
||||
|
||||
// Also verify that contrastive variants share the split_group with
|
||||
// their base seed from ru_routing_v1.
|
||||
ruGroups := map[string]string{} // source_id → split_group
|
||||
for _, e := range exs {
|
||||
if e.Source == "ru_routing_v1" {
|
||||
ruGroups[e.SourceID] = e.SplitGroup
|
||||
}
|
||||
}
|
||||
for sourceID, groups := range baseGroups {
|
||||
var contrastGroup string
|
||||
for g := range groups {
|
||||
contrastGroup = g
|
||||
}
|
||||
if ruGroup, ok := ruGroups[sourceID]; ok && contrastGroup != ruGroup {
|
||||
t.Errorf("base %q: ru_routing_v1 split_group=%q, contrastive split_group=%q",
|
||||
sourceID, ruGroup, contrastGroup)
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("contrast family leakage check passed: %d base seeds, all groups consistent",
|
||||
len(baseGroups))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
package semantic
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/kami/maven/internal/router"
|
||||
)
|
||||
|
||||
const seedDir = "../../../models/seeds"
|
||||
|
||||
var seedIntents = []router.Intent{
|
||||
router.IntentAct, router.IntentReminder, router.IntentFact,
|
||||
router.IntentNote, router.IntentQuery, router.IntentChat, router.IntentSystem,
|
||||
}
|
||||
|
||||
func actVerbList() []string {
|
||||
return []string{
|
||||
"перезапусти", "перезагрузи", "выключи", "включи", "останови", "запусти",
|
||||
"закрой", "открой", "сделай", "поставь",
|
||||
"restart", "reboot", "stop", "start", "turn off", "turn on", "open", "close",
|
||||
}
|
||||
}
|
||||
|
||||
func buildSeededClassifier(t *testing.T, emb router.Embedder) *router.Classifier {
|
||||
t.Helper()
|
||||
cls := router.NewClassifier(emb)
|
||||
ctx := context.Background()
|
||||
seeds := seedsWithIntent(t)
|
||||
texts := make([]string, 0, len(seeds))
|
||||
for text := range seeds {
|
||||
texts = append(texts, text)
|
||||
}
|
||||
sort.Strings(texts)
|
||||
for _, text := range texts {
|
||||
if err := cls.AddExample(ctx, seeds[text], text); err != nil {
|
||||
t.Fatalf("seed %q: %v", text, err)
|
||||
}
|
||||
}
|
||||
return cls
|
||||
}
|
||||
|
||||
func seedsWithIntent(t *testing.T) map[string]router.Intent {
|
||||
t.Helper()
|
||||
files := readSeedFiles(t)
|
||||
out := map[string]router.Intent{}
|
||||
for _, intent := range seedIntents {
|
||||
for _, l := range files[intent] {
|
||||
if _, dup := out[l]; dup {
|
||||
continue
|
||||
}
|
||||
out[l] = intent
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func readSeedFiles(t *testing.T) map[router.Intent][]string {
|
||||
t.Helper()
|
||||
out := map[router.Intent][]string{}
|
||||
for _, intent := range seedIntents {
|
||||
path := filepath.Join(seedDir, string(intent)+".txt")
|
||||
fh, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open %s: %v", path, err)
|
||||
}
|
||||
sc := bufio.NewScanner(fh)
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
out[intent] = append(out[intent], line)
|
||||
}
|
||||
err = sc.Err()
|
||||
fh.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
Reference in New Issue
Block a user